From d41b48849561a258249e8117340a2616aa4ca252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 10 Apr 2025 13:02:17 +0300 Subject: [PATCH 1/3] MDEV-21923: LSN allocation is a bottleneck The parameter innodb_log_spin_wait_delay will be deprecated and ignored, because there is no spin loop anymore. Thanks to commit 685d958e38b825ad9829be311f26729cccf37c46 and commit a635c40648519fd6c3729c9657872a16a0a20821 multiple mtr_t::commit() can concurrently copy their slice of mtr_t::m_log to the shared log_sys.buf. Each writer would allocate their own log sequence number by invoking log_t::append_prepare() while holding a shared log_sys.latch. This function was too heavy, because it would invoke a minimum of 4 atomic read-modify-write operations as well as system calls in the supposedly fast code path. It turns out that with a simpler data structure, instead of having several data fields that needed to be kept consistent with each other, we only need one Atomic_relaxed write_lsn_offset, on which we can operate using fetch_add(), fetch_sub() as well as a single-bit fetch_or(), which reasonably modern compilers (GCC 7, Clang 15 or later) can translate into loop-free code on AMD64. Before anything can be written to the log, log_sys.clear_mmap() must be invoked. log_t::base_lsn: The LSN of the last write_buf() or persist(). This is a rough approximation of log_sys.lsn, which will be removed. log_t::write_lsn_offset: An Atomic_relaxed that buffers updates of write_to_buf and base_lsn. log_t::buf_free, log_t::max_buf_free, log_t::lsn. Remove. Replaced by base_lsn and write_lsn_offset. log_t::buf_size: Always reflects the usable size in append_prepare(). log_t::lsn_lock: Remove. For the memory-mapped log in resize_write(), there will be a resize_wrap_mutex. log_t::get_lsn_approx(): Return a lower bound of get_lsn(). This should be exact unless append_prepare_wait() is pending. log_get_lsn(): A wrapper for log_sys.get_lsn(), which must be invoked while holding an exclusive log_sys.latch. recv_recovery_from_checkpoint_start(): Do not invoke fil_names_clear(); it would seem to be unnecessary. In many places, references to log_sys.get_lsn() are replaced with log_sys.get_flushed_lsn(), which remains a simple std::atomic::load(). Reviewed by: Debarun Banerjee (cherry picked from commit acd071f599f416ddb4821dec485c4d912844213f) --- extra/mariabackup/xtrabackup.cc | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/buf/buf0buf.cc | 18 +- storage/innobase/buf/buf0dblwr.cc | 5 +- storage/innobase/buf/buf0flu.cc | 41 +-- storage/innobase/handler/ha_innodb.cc | 58 ++-- storage/innobase/include/fil0fil.h | 2 +- storage/innobase/include/log0log.h | 174 ++++++------ storage/innobase/include/mtr0mtr.h | 9 +- storage/innobase/log/log0crypt.cc | 2 +- storage/innobase/log/log0log.cc | 204 +++++++------- storage/innobase/log/log0recv.cc | 30 +- storage/innobase/mtr/mtr0mtr.cc | 265 ++++++------------ storage/innobase/row/row0mysql.cc | 4 +- storage/innobase/srv/srv0mon.cc | 8 +- storage/innobase/srv/srv0srv.cc | 6 +- storage/innobase/srv/srv0start.cc | 23 +- 17 files changed, 383 insertions(+), 470 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 3aba403e74dd3..1af5cdf87baa6 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2672,7 +2672,7 @@ static bool innodb_init() } recv_sys.lsn= log_sys.next_checkpoint_lsn= - log_sys.get_lsn() - SIZE_OF_FILE_CHECKPOINT; + log_get_lsn() - SIZE_OF_FILE_CHECKPOINT; log_sys.set_latest_format(false); // not encrypted log_hdr_init(); byte *b= &log_hdr_buf[log_t::START_OFFSET]; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index c6f21371c8b43..1043684ba2c34 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1008,7 +1008,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Delay between log buffer spin lock polls (0 to use a blocking latch) +VARIABLE_COMMENT Deprecated parameter with no effect NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6000 NUMERIC_BLOCK_SIZE 0 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index e2d2a2e1a943c..fd0617ca4ee78 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -557,16 +557,18 @@ buf_page_is_checksum_valid_crc32( } #ifndef UNIV_INNOCHECKSUM -/** Checks whether the lsn present in the page is lesser than the -peek current lsn. -@param check_lsn lsn to check +/** Check whether a page is newer than the durable LSN. +@param check_lsn whether to check the LSN @param read_buf page frame -@return whether the FIL_PAGE_LSN is invalid */ -static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) +@return whether the FIL_PAGE_LSN is invalid (ahead of the durable LSN) */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) noexcept { if (!check_lsn) return false; - lsn_t current_lsn= log_sys.get_lsn(); + /* A page may not be read before it is written, and it may not be + written before the corresponding log has been durably written. + Hence, we refer to the current durable LSN here */ + lsn_t current_lsn= log_sys.get_flushed_lsn(std::memory_order_relaxed); if (UNIV_UNLIKELY(current_lsn == log_sys.FIRST_LSN) && srv_force_recovery == SRV_FORCE_NO_LOG_REDO) return false; @@ -2651,8 +2653,8 @@ buf_page_get_gen( innodb_undo_tablespaces=127. */ ut_d(extern bool ibuf_upgrade_was_needed;) ut_ad(mode == BUF_GET_RECOVER - ? recv_recovery_is_on() || log_sys.get_lsn() < 120000 - || log_sys.get_lsn() == recv_sys.lsn + SIZE_OF_FILE_CHECKPOINT + ? recv_recovery_is_on() || log_get_lsn() < 120000 + || log_get_lsn() == recv_sys.lsn + SIZE_OF_FILE_CHECKPOINT || ibuf_upgrade_was_needed : !recv_recovery_is_on() || recv_sys.after_apply); ut_ad(mtr->is_active()); diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index aca07caadb367..4fad98cb2471e 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -366,7 +366,7 @@ void buf_dblwr_t::recover() noexcept ut_ad(log_sys.last_checkpoint_lsn); if (!is_created()) return; - const lsn_t max_lsn{log_sys.get_lsn()}; + const lsn_t max_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; ut_ad(recv_sys.scanned_lsn == max_lsn); ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); @@ -785,6 +785,9 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); log_write_up_to(lsn, true); + ut_ad(!e.request.node->space->full_crc32() || + !buf_page_is_corrupted(true, static_cast(frame), + e.request.node->space->flags)); e.request.node->space->io(e.request, bpage->physical_offset(), e_size, frame, bpage); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index b204abff41e99..65c47fe4801d6 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -694,7 +694,6 @@ static byte *buf_page_encrypt(fil_space_t *space, buf_page_t *bpage, byte *s, { static_assert(FIL_PAGE_FCRC32_CHECKSUM == 4, "alignment"); mach_write_to_4(tmp + len - 4, my_crc32c(0, tmp, len - 4)); - ut_ad(!buf_page_is_corrupted(true, tmp, space->flags)); } d= tmp; @@ -854,6 +853,8 @@ bool buf_page_t::flush(fil_space_t *space) noexcept if (!space->is_temporary() && !space->is_being_imported() && lsn > log_sys.get_flushed_lsn()) log_write_up_to(lsn, true); + ut_ad(space->is_temporary() || !space->full_crc32() || + !buf_page_is_corrupted(true, write_frame, space->flags)); space->io(IORequest{type, this, slot}, physical_offset(), size, write_frame, this); } @@ -1771,8 +1772,9 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept { ut_ad(!srv_read_only_mode); ut_ad(end_lsn >= next_checkpoint_lsn); - ut_ad(end_lsn <= get_lsn()); - ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= get_lsn() || + ut_d(const lsn_t current_lsn{get_lsn()}); + ut_ad(end_lsn <= current_lsn); + ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= current_lsn || srv_shutdown_state > SRV_SHUTDOWN_INITIATED); DBUG_PRINT("ib_log", @@ -1901,7 +1903,8 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(!is_opened()); my_munmap(buf, file_size); buf= resize_buf; - set_buf_free(START_OFFSET + (get_lsn() - resizing)); + buf_size= unsigned(std::min(resize_target - START_OFFSET, + buf_size_max)); } else #endif @@ -2030,9 +2033,9 @@ static bool log_checkpoint() noexcept } /** Make a checkpoint. */ -ATTRIBUTE_COLD void log_make_checkpoint() +ATTRIBUTE_COLD void log_make_checkpoint() noexcept { - buf_flush_wait_flushed(log_sys.get_lsn(std::memory_order_acquire)); + buf_flush_wait_flushed(log_get_lsn()); while (!log_checkpoint()); } @@ -2040,8 +2043,6 @@ ATTRIBUTE_COLD void log_make_checkpoint() NOTE: The calling thread is not allowed to hold any buffer page latches! */ static void buf_flush_wait(lsn_t lsn) noexcept { - ut_ad(lsn <= log_sys.get_lsn()); - lsn_t oldest_lsn; while ((oldest_lsn= buf_pool.get_oldest_modification(lsn)) < lsn) @@ -2249,7 +2250,7 @@ redo log capacity filled threshold. @return true if adaptive flushing is recommended. */ static bool af_needed_for_redo(lsn_t oldest_lsn) noexcept { - lsn_t age= (log_sys.get_lsn() - oldest_lsn); + lsn_t age= log_sys.get_lsn_approx() - oldest_lsn; lsn_t af_lwm= static_cast(srv_adaptive_flushing_lwm * static_cast(log_sys.log_capacity) / 100); @@ -2309,7 +2310,7 @@ static ulint page_cleaner_flush_pages_recommendation(ulint last_pages_in, lsn_t lsn_rate; ulint n_pages = 0; - const lsn_t cur_lsn = log_sys.get_lsn(); + const lsn_t cur_lsn = log_sys.get_lsn_approx(); ut_ad(oldest_lsn <= cur_lsn); ulint pct_for_lsn = af_get_pct_for_lsn(cur_lsn - oldest_lsn); time_t curr_time = time(nullptr); @@ -2782,7 +2783,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() noexcept NOTE: The calling thread is not allowed to hold any buffer page latches! */ void buf_flush_sync_batch(lsn_t lsn) noexcept { - lsn= std::max(lsn, log_sys.get_lsn()); + lsn= std::max(lsn, log_get_lsn()); mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -2801,20 +2802,26 @@ void buf_flush_sync() noexcept thd_wait_begin(nullptr, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - for (;;) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + + for (lsn_t lsn= log_sys.get_lsn();;) { - const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); /* Wait for the page cleaner to be idle (for log resizing at startup) */ while (buf_flush_sync_lsn) my_cond_wait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex.m_mutex); - if (lsn == log_sys.get_lsn()) + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t new_lsn= log_sys.get_lsn(); + if (lsn == new_lsn) break; + lsn= new_lsn; } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_unlock(); tpool::tpool_wait_end(); thd_wait_end(nullptr); } @@ -2834,7 +2841,7 @@ ATTRIBUTE_COLD void buf_pool_t::print_flush_info() const noexcept "-------------------", lru_size, free_size, dirty_size, dirty_pct); - lsn_t lsn= log_sys.get_lsn(); + lsn_t lsn= log_get_lsn(); lsn_t clsn= log_sys.last_checkpoint_lsn; sql_print_information("InnoDB: LSN flush parameters\n" "-------------------\n" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d26a8ca734cd0..41c564776f751 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -243,11 +243,11 @@ static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, if (thd_kill_level(thd)) break; /* Adjust for purge_coordinator_state::refresh() */ - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; - log_sys.latch.rd_unlock(); const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); @@ -1136,7 +1136,7 @@ innobase_rollback_to_savepoint_can_release_mdl( be rolled back to savepoint */ /** Request notification of log writes */ -static void innodb_log_flush_request(void *cookie); +static void innodb_log_flush_request(void *cookie) noexcept; /** Requests for log flushes */ struct log_flush_request @@ -4771,11 +4771,13 @@ void log_flush_notify(lsn_t flush_lsn) We put the request in a queue, so that we can notify upper layer about checkpoint complete when we have flushed the redo log. If we have already flushed all relevant redo log, we notify immediately.*/ -static void innodb_log_flush_request(void *cookie) +static void innodb_log_flush_request(void *cookie) noexcept { + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn_t flush_lsn= log_sys.get_flushed_lsn(); /* Load lsn relaxed after flush_lsn was loaded from the same cache line */ const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if (flush_lsn >= lsn) /* All log is already persistent. */; @@ -18428,11 +18430,16 @@ checkpoint_now_set(THD* thd, st_mysql_sys_var*, void*, const void *save) const auto size= log_sys.is_encrypted() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; mysql_mutex_unlock(&LOCK_global_system_variables); - lsn_t lsn; - while (!thd_kill_level(thd) && - log_sys.last_checkpoint_lsn.load(std::memory_order_acquire) + size < - (lsn= log_sys.get_lsn(std::memory_order_acquire))) + while (!thd_kill_level(thd)) + { + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t cp= log_sys.last_checkpoint_lsn.load(std::memory_order_relaxed), + lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + if (cp + size >= lsn) + break; log_make_checkpoint(); + } mysql_mutex_lock(&LOCK_global_system_variables); } @@ -18683,35 +18690,23 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, mysql_mutex_unlock(&buf_pool.flush_list_mutex); if (!resizing || !log_sys.resize_running(thd)) break; - if (resizing > log_sys.get_lsn()) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + while (resizing > log_sys.get_lsn()) { ut_ad(!log_sys.is_mmap()); /* The server is almost idle. Write dummy FILE_CHECKPOINT records to ensure that the log resizing will complete. */ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - while (resizing > log_sys.get_lsn()) - { - mtr_t mtr; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); + mtr_t mtr; + mtr.start(); + mtr.commit_files(log_sys.last_checkpoint_lsn); } + log_sys.latch.wr_unlock(); } } } mysql_mutex_lock(&LOCK_global_system_variables); } -static void innodb_log_spin_wait_delay_update(THD *, st_mysql_sys_var*, - void *, const void *save) -{ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - mtr_t::spin_wait_delay= *static_cast(save); - mtr_t::finisher_update(); - log_sys.latch.wr_unlock(); -} - /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -19545,11 +19540,12 @@ static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, nullptr, innodb_log_file_size_update, 96 << 20, 4 << 20, std::numeric_limits::max(), 4096); -static MYSQL_SYSVAR_UINT(log_spin_wait_delay, mtr_t::spin_wait_delay, - PLUGIN_VAR_OPCMDARG, - "Delay between log buffer spin lock polls (0 to use a blocking latch)", - nullptr, innodb_log_spin_wait_delay_update, - 0, 0, 6000, 0); +static uint innodb_log_spin_wait_delay; + +static MYSQL_SYSVAR_UINT(log_spin_wait_delay, innodb_log_spin_wait_delay, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_DEPRECATED, + "Deprecated parameter with no effect", + nullptr, nullptr, 0, 0, 6000, 0); static MYSQL_SYSVAR_UINT(old_blocks_pct, innobase_old_blocks_pct, PLUGIN_VAR_RQCMDARG, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index d9b92764eac1e..a421ed2f03e52 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -319,7 +319,7 @@ struct fil_space_t final /** fil_system.spaces chain node */ fil_space_t *hash= nullptr; /** log_sys.get_lsn() of the most recent fil_names_write_if_was_clean(). - Reset to 0 by fil_names_clear(). Protected by log_sys.mutex. + Reset to 0 by fil_names_clear(). Protected by log_sys.latch_have_wr(). If and only if this is nonzero, the tablespace will be in named_spaces. */ lsn_t max_lsn= 0; /** base node for the chain of data files; multiple entries are diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 6a6d722f135a3..ce6a4f2779456 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -64,20 +64,19 @@ void log_write_up_to(lsn_t lsn, bool durable, /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable= true); - +void log_buffer_flush_to_disk(bool durable= true) noexcept; /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare(); +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept; /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush(); +ATTRIBUTE_COLD void log_write_and_flush() noexcept; /** Make a checkpoint */ -ATTRIBUTE_COLD void log_make_checkpoint(); +ATTRIBUTE_COLD void log_make_checkpoint() noexcept; /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown(); +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept; /******************************************************//** Prints info of the log. */ @@ -167,40 +166,35 @@ struct log_t static constexpr lsn_t FIRST_LSN= START_OFFSET; private: - /** the lock bit in buf_free */ - static constexpr size_t buf_free_LOCK= ~(~size_t{0} >> 1); + /** the least significant bit of the write_to_buf buffer */ + static constexpr size_t WRITE_TO_BUF_SHIFT{34}; + /** write_lsn_offset component for incrementing write_to_buf */ + static constexpr uint64_t WRITE_TO_BUF{1ULL << WRITE_TO_BUF_SHIFT}; + /** write_lsn_offset flag to indicate that append_prepare_wait() is active */ + static constexpr uint64_t WRITE_BACKOFF{1ULL << 33}; + + /** The current log sequence number, relative to base_lsn, and flags; + may be modified while latch_have_any() */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) - /** first free offset within buf used; - the most significant bit is set by lock_lsn() to protect this field - as well as write_to_buf, waits */ - std::atomic buf_free; -public: - /** number of write requests (to buf); protected by lock_lsn() or lsn_lock */ - size_t write_to_buf; - /** log record buffer, written to by mtr_t::commit() */ - byte *buf; -private: - /** The log sequence number of the last change of durable InnoDB files; - protected by lock_lsn() or lsn_lock or latch.wr_lock() */ - std::atomic lsn; + Atomic_relaxed write_lsn_offset; + /** the LSN of the last write_buf() or persist(); protected by latch */ + std::atomic base_lsn; /** the first guaranteed-durable log sequence number */ std::atomic flushed_to_disk_lsn; public: - /** number of append_prepare_wait(); protected by lock_lsn() or lsn_lock */ - size_t waits; - /** innodb_log_buffer_size (size of buf,flush_buf if !is_mmap(), in bytes) */ + /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ unsigned buf_size; /** log file size in bytes, including the header */ lsn_t file_size; #ifdef LOG_LATCH_DEBUG typedef srw_lock_debug log_rwlock; - typedef srw_mutex log_lsn_lock; bool latch_have_wr() const { return latch.have_wr(); } bool latch_have_rd() const { return latch.have_rd(); } bool latch_have_any() const { return latch.have_any(); } #else + typedef srw_lock log_rwlock; # ifndef UNIV_DEBUG # elif defined SUX_LOCK_GENERIC bool latch_have_wr() const { return true; } @@ -211,23 +205,23 @@ struct log_t bool latch_have_rd() const { return latch.is_locked(); } bool latch_have_any() const { return latch.is_locked(); } # endif -# ifdef __aarch64__ - /* On ARM, we spin more */ - typedef srw_spin_lock log_rwlock; - typedef pthread_mutex_wrapper log_lsn_lock; -# else - typedef srw_lock log_rwlock; - typedef srw_mutex log_lsn_lock; -# endif #endif - /** exclusive latch for checkpoint, shared for mtr_t::commit() to buf */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch; + /** latch_have_wr() for checkpoint, latch_have_any() for append_prepare() */ + log_rwlock latch; + + /** log record buffer, written to by mtr_t::commit() */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) byte *buf; + + /** number of write requests to buf, + excluding (write_lsn_offset & WRITE_TO_BUF); + protected by latch.wr_lock() */ + size_t write_to_buf; /** number of writes from buf or flush_buf to log; protected by latch.wr_lock() */ - ulint write_to_log; + size_t write_to_log; - /** Last written LSN */ + /** Last written LSN; protected by latch */ lsn_t write_lsn; /** Buffer for writing data to ib_logfile0, or nullptr if is_mmap(). @@ -241,8 +235,6 @@ struct log_t Atomic_relaxed checkpoint_pending; /** next checkpoint number (protected by latch.wr_lock()) */ byte next_checkpoint_no; - /** recommended maximum buf_free size, after which the buffer is flushed */ - unsigned max_buf_free; /** Log sequence number when a log file overwrite (broken crash recovery) was noticed. Protected by latch.wr_lock(). */ lsn_t overwrite_warned; @@ -266,12 +258,6 @@ struct log_t /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; - /** Special implementation of lock_lsn() for IA-32 and AMD64 */ - void lsn_lock_bts() noexcept; - /** Acquire a lock for updating buf_free and related fields. - @return the value of buf_free */ - size_t lock_lsn() noexcept; - /** log sequence number when log resizing was initiated; 0 if the log is not being resized, 1 if resize_start() is in progress */ std::atomic resize_lsn; @@ -330,34 +316,24 @@ struct log_t private: /** the thread that initiated resize_lsn() */ Atomic_relaxed resize_initiator; - /** A lock when the spin-only lock_lsn() is not being used */ - log_lsn_lock lsn_lock; +#ifdef HAVE_PMEM + /** mutex protecting wrap-around in resize_write() */ + srw_mutex resize_wrap_mutex; +#endif public: + /** number of long append_prepare_wait(); protected by latch_have_wr() */ + size_t waits; - bool is_initialised() const noexcept { return max_buf_free != 0; } - - /** whether there is capacity in the log buffer */ - bool buf_free_ok() const noexcept - { - ut_ad(!is_mmap()); - return (buf_free.load(std::memory_order_relaxed) & ~buf_free_LOCK) < - max_buf_free; - } - + bool is_initialised() const noexcept + { return base_lsn.load(std::memory_order_relaxed) != 0; } inline void set_recovered() noexcept; - void set_buf_free(size_t f) noexcept - { ut_ad(f < buf_free_LOCK); buf_free.store(f, std::memory_order_relaxed); } - bool is_mmap() const noexcept { return !flush_buf; } /** @return whether a handle to the log is open; is_mmap() && !is_opened() holds for PMEM */ bool is_opened() const noexcept { return log.is_opened(); } - /** @return target write LSN to react on !buf_free_ok() */ - inline lsn_t get_write_target() const; - /** @return LSN at which log resizing was started and is still in progress @retval 0 if no log resizing is in progress @retval 1 if resize_start() is in progress */ @@ -410,55 +386,66 @@ struct log_t { return resize_buf + resize_target; } /** Initialise the redo log subsystem. */ - void create(); + void create() noexcept; /** Attach a log file. @return whether the memory allocation succeeded */ - bool attach(log_file_t file, os_offset_t size); + bool attach(log_file_t file, os_offset_t size) noexcept; /** Disable memory-mapped access (update log_mmap) */ - void clear_mmap(); - void close_file(bool really_close= true); + void clear_mmap() noexcept; + void close_file(bool really_close= true) noexcept; #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ - void set_buffered(bool buffered); + void set_buffered(bool buffered) noexcept; #endif /** Try to enable or disable durable writes (update log_write_through) */ void set_write_through(bool write_through); /** Calculate the checkpoint safety margins. */ - static void set_capacity(); + static void set_capacity() noexcept; /** Write a log file header. @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ - static void header_write(byte *buf, lsn_t lsn, bool encrypted); + static void header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept; + + /** @return a lower bound estimate of get_lsn(), + using acquire-release ordering with write_buf() or persist(); + this is exact unless append_prepare_wait() is pending */ + lsn_t get_lsn_approx() const noexcept + { + /* acquire-release ordering with write_buf() and persist() */ + lsn_t lsn= base_lsn.load(std::memory_order_acquire); + lsn += write_lsn_offset.load(std::memory_order_relaxed) & + (WRITE_BACKOFF - 1); + return lsn; + } - lsn_t get_lsn(std::memory_order order= std::memory_order_relaxed) const - { return lsn.load(order); } + /** @return the current log sequence number (logical time stamp) */ + lsn_t get_lsn() const noexcept + { + ut_ad(latch_have_wr()); + return base_lsn.load(std::memory_order_relaxed) + + (write_lsn_offset & (WRITE_BACKOFF - 1)); + } lsn_t get_flushed_lsn(std::memory_order order= std::memory_order_acquire) const noexcept { return flushed_to_disk_lsn.load(order); } /** Initialize the LSN on initial log file creation. */ - lsn_t init_lsn() noexcept - { - latch.wr_lock(SRW_LOCK_CALL); - const lsn_t lsn{get_lsn()}; - flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); - write_lsn= lsn; - latch.wr_unlock(); - return lsn; - } + inline lsn_t init_lsn() noexcept; void set_recovered_lsn(lsn_t lsn) noexcept { ut_ad(latch_have_wr()); - write_lsn= lsn; - this->lsn.store(lsn, std::memory_order_relaxed); + uint64_t lsn_offset= ((write_size - 1) & (lsn - first_lsn)); + write_lsn_offset= lsn_offset; + base_lsn.store(lsn - lsn_offset, std::memory_order_relaxed); flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; } #ifdef HAVE_PMEM @@ -496,22 +483,16 @@ struct log_t void writer_update(bool resizing) noexcept; /** Wait in append_prepare() for buffer to become available - @tparam spin whether to use the spin-only lock_lsn() - @param b the value of buf_free - @param ex whether log_sys.latch is exclusively locked - @param lsn log sequence number to write up to - @return the new value of buf_free */ - template - ATTRIBUTE_COLD size_t append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept; + @param late whether the WRITE_BACKOFF flag had already been set + @param ex whether log_sys.latch is exclusively locked */ + ATTRIBUTE_COLD void append_prepare_wait(bool late, bool ex) noexcept; public: /** Reserve space in the log buffer for appending data. - @tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ - template + template std::pair append_prepare(size_t size, bool ex) noexcept; /** Append a string of bytes to the redo log. @@ -582,7 +563,10 @@ extern log_t log_sys; /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check(); +void log_free_check() noexcept; + +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept; /** Release the latches that protect log resizing. */ -void log_resize_release(); +void log_resize_release() noexcept; diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 7acbd49220033..ac5aafc87a57e 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -694,19 +694,19 @@ struct mtr_t { @param mtr mini-transaction @param lsns {start_lsn,flush_ahead} */ template - static void commit_log(mtr_t *mtr, std::pair lsns); + static void commit_log(mtr_t *mtr, std::pair lsns) + noexcept; /** Append the redo log records to the redo log buffer. @return {start_lsn,flush_ahead} */ std::pair do_write(); /** Append the redo log records to the redo log buffer. - @tparam spin whether to use the spin-only log_sys.lock_lsn() @tparam mmap log_sys.is_mmap() @param mtr mini-transaction @param len number of bytes to write @return {start_lsn,flush_ahead} */ - template static + template static std::pair finish_writer(mtr_t *mtr, size_t len); /** The applicable variant of commit_log() */ @@ -717,9 +717,6 @@ struct mtr_t { std::pair finish_write(size_t len) { return finisher(this, len); } public: - /** Poll interval in log_sys.lock_lsn(); 0 to use log_sys.lsn_lock. - Protected by LOCK_global_system_variables and log_sys.latch. */ - static unsigned spin_wait_delay; /** Update finisher when spin_wait_delay is changing to or from 0. */ static void finisher_update(); private: diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index fc7a51807979b..42591e39baa64 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -566,7 +566,7 @@ ATTRIBUTE_NOINLINE void mtr_t::encrypt() alignas(8) byte iv[MY_AES_BLOCK_SIZE]; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); ut_ad(m_commit_lsn); byte *tmp= static_cast(alloca(srv_page_size)), *t= tmp; byte *dst= static_cast(alloca(srv_page_size)); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index dffb26cf06ecf..1bf47bfe14eea 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -67,7 +67,7 @@ log_t log_sys; #define LOG_BUF_FLUSH_MARGIN ((4 * 4096) /* cf. log_t::append_prepare() */ \ + (4U << srv_page_size_shift)) -void log_t::set_capacity() +void log_t::set_capacity() noexcept { ut_ad(log_sys.latch_have_wr()); /* Margin for the free space in the smallest log, before a new query @@ -86,13 +86,15 @@ void log_t::set_capacity() log_sys.max_checkpoint_age = margin; } -void log_t::create() +void log_t::create() noexcept { ut_ad(this == &log_sys); ut_ad(!is_initialised()); + latch.SRW_LOCK_INIT(log_latch_key); + write_lsn_offset= 0; /* LSN 0 and 1 are reserved; @see buf_page_t::oldest_modification_ */ - lsn.store(FIRST_LSN, std::memory_order_relaxed); + base_lsn.store(FIRST_LSN, std::memory_order_relaxed); flushed_to_disk_lsn.store(FIRST_LSN, std::memory_order_relaxed); need_checkpoint.store(true, std::memory_order_relaxed); write_lsn= FIRST_LSN; @@ -101,10 +103,10 @@ void log_t::create() ut_ad(!buf); ut_ad(!flush_buf); ut_ad(!writer); - max_buf_free= 1; - latch.SRW_LOCK_INIT(log_latch_key); - lsn_lock.init(); +#ifdef HAVE_PMEM + resize_wrap_mutex.init(); +#endif last_checkpoint_lsn= FIRST_LSN; log_capacity= 0; @@ -113,8 +115,6 @@ void log_t::create() next_checkpoint_lsn= 0; checkpoint_pending= false; - set_buf_free(0); - ut_ad(is_initialised()); } @@ -305,7 +305,7 @@ static void *log_mmap(os_file_t file, #if defined __linux__ || defined _WIN32 /** Display a message about opening the log */ -ATTRIBUTE_COLD static void log_file_message() +ATTRIBUTE_COLD static void log_file_message() noexcept { sql_print_information("InnoDB: %s (block size=%u bytes)", log_sys.log_mmap @@ -319,10 +319,10 @@ ATTRIBUTE_COLD static void log_file_message() log_sys.write_size); } #else -static inline void log_file_message() {} +static inline void log_file_message() noexcept {} #endif -bool log_t::attach(log_file_t file, os_offset_t size) +bool log_t::attach(log_file_t file, os_offset_t size) noexcept { log= file; ut_ad(!size || size >= START_OFFSET + SIZE_OF_FILE_CHECKPOINT); @@ -351,7 +351,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) } # endif buf= static_cast(ptr); - max_buf_free= 1; writer_update(false); # ifdef HAVE_PMEM if (is_pmem) @@ -365,7 +364,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) if (!buf) { alloc_fail: - max_buf_free= 0; + base_lsn.store(0, std::memory_order_relaxed); sql_print_error("InnoDB: Cannot allocate memory;" " too large innodb_log_buffer_size?"); return false; @@ -393,7 +392,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); - max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; writer_update(false); memset_aligned<512>(checkpoint_buf, 0, write_size); @@ -406,7 +404,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ -void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) +void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept { mach_write_to_4(my_assume_aligned<4>(buf) + LOG_HEADER_FORMAT, log_sys.FORMAT_10_8); @@ -435,8 +433,9 @@ void log_t::create(lsn_t lsn) noexcept ut_ad(is_latest()); ut_ad(this == &log_sys); - this->lsn.store(lsn, std::memory_order_relaxed); - this->flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn_offset= 0; + base_lsn.store(lsn, std::memory_order_relaxed); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); first_lsn= lsn; write_lsn= lsn; @@ -451,14 +450,13 @@ void log_t::create(lsn_t lsn) noexcept mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); memset_aligned<4096>(buf, 0, 4096); log_sys.header_write(buf, lsn, is_encrypted()); - set_buf_free(START_OFFSET); pmem_persist(buf, 512); + buf_size= unsigned(std::min(capacity(), buf_size_max)); } else #endif { ut_ad(!is_mmap()); - set_buf_free(0); memset_aligned<4096>(flush_buf, 0, buf_size); memset_aligned<4096>(buf, 0, buf_size); log_sys.header_write(buf, lsn, is_encrypted()); @@ -467,12 +465,12 @@ void log_t::create(lsn_t lsn) noexcept } } -ATTRIBUTE_COLD static void log_close_failed(dberr_t err) +ATTRIBUTE_COLD static void log_close_failed(dberr_t err) noexcept { ib::fatal() << "closing ib_logfile0 failed: " << err; } -void log_t::close_file(bool really_close) +void log_t::close_file(bool really_close) noexcept { if (is_mmap()) { @@ -507,16 +505,25 @@ void log_t::close_file(bool really_close) log_close_failed(err); } +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept +{ + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + return lsn; +} + /** Acquire all latches that protect the log. */ -static void log_resize_acquire() +static void log_resize_acquire() noexcept { #ifdef HAVE_PMEM if (!log_sys.is_mmap()) #endif { - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } @@ -524,7 +531,7 @@ static void log_resize_acquire() } /** Release the latches that protect the log. */ -void log_resize_release() +void log_resize_release() noexcept { log_sys.latch.wr_unlock(); @@ -541,7 +548,7 @@ void log_resize_release() #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ -void log_t::set_buffered(bool buffered) +void log_t::set_buffered(bool buffered) noexcept { if (!log_maybe_unbuffered || #ifdef HAVE_PMEM @@ -911,9 +918,7 @@ void log_t::persist(lsn_t lsn) noexcept ut_ad(!is_opened()); ut_ad(!write_lock.is_owner()); ut_ad(!flush_lock.is_owner()); -#ifdef LOG_LATCH_DEBUG - ut_ad(latch_have_any()); -#endif + ut_ad(latch_have_wr()); lsn_t old= flushed_to_disk_lsn.load(std::memory_order_relaxed); @@ -931,26 +936,26 @@ void log_t::persist(lsn_t lsn) noexcept else pmem_persist(buf + start, end - start); - old= flushed_to_disk_lsn.load(std::memory_order_relaxed); - - if (old < lsn) - { - while (!flushed_to_disk_lsn.compare_exchange_weak - (old, lsn, std::memory_order_release, std::memory_order_relaxed)) - if (old >= lsn) - break; - - log_flush_notify(lsn); - DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); - } + uint64_t offset{write_lsn_offset}; + const lsn_t new_base_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + ut_ad(new_base_lsn >= lsn); + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(new_base_lsn, std::memory_order_release); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + log_flush_notify(lsn); + DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); } ATTRIBUTE_NOINLINE static void log_write_persist(lsn_t lsn) noexcept { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); log_sys.persist(lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); } #endif @@ -1001,7 +1006,7 @@ lsn_t log_t::write_buf() noexcept ut_ad(resizing == RETAIN_LATCH || (resizing == RESIZING) == (resize_in_progress() > 1)); - const lsn_t lsn{get_lsn(std::memory_order_relaxed)}; + const lsn_t lsn{get_lsn()}; if (write_lsn >= lsn) { @@ -1017,7 +1022,8 @@ lsn_t log_t::write_buf() noexcept ut_ad(write_lsn >= get_flushed_lsn()); const size_t write_size_1{write_size - 1}; ut_ad(ut_is_2pow(write_size)); - size_t length{buf_free.load(std::memory_order_relaxed)}; + lsn_t base= base_lsn.load(std::memory_order_relaxed); + size_t length{size_t(lsn - base)}; lsn_t offset{calc_lsn_offset(write_lsn)}; ut_ad(length >= (offset & write_size_1)); ut_ad(write_size_1 >= 511); @@ -1039,14 +1045,8 @@ lsn_t log_t::write_buf() noexcept { ut_ad(!((length ^ (size_t(lsn) - size_t(first_lsn))) & write_size_1)); /* Keep filling the same buffer until we have more than one block. */ -#if 0 /* TODO: Pad the last log block with dummy records. */ - buf_free= log_pad(lsn, (write_size_1 + 1) - length, - buf + length, flush_buf); - ... /* TODO: Update the LSN and adjust other code. */ -#else MEM_MAKE_DEFINED(buf + length, (write_size_1 + 1) - length); buf[length]= 0; /* ensure that recovery catches EOF */ -#endif if (UNIV_LIKELY_NULL(re_write_buf)) { MEM_MAKE_DEFINED(re_write_buf + length, (write_size_1 + 1) - length); @@ -1057,8 +1057,13 @@ lsn_t log_t::write_buf() noexcept else { const size_t new_buf_free{length & write_size_1}; + base+= length & ~write_size_1; ut_ad(new_buf_free == ((lsn - first_lsn) & write_size_1)); - buf_free.store(new_buf_free, std::memory_order_relaxed); + write_to_buf+= size_t(write_lsn_offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(new_buf_free, std::memory_order_relaxed); + base_lsn.store(base, std::memory_order_release); if (new_buf_free) { @@ -1087,7 +1092,9 @@ lsn_t log_t::write_buf() noexcept std::swap(resize_buf, resize_flush_buf); } + ut_ad(base + (write_lsn_offset & (WRITE_TO_BUF - 1)) == lsn); write_to_log++; + if (resizing != RETAIN_LATCH) latch.wr_unlock(); @@ -1131,7 +1138,7 @@ bool log_t::flush(lsn_t lsn) noexcept @retval 0 if there are no pending callbacks on flush_lock or there is another group commit lead. */ -static lsn_t log_flush(lsn_t lsn) +static lsn_t log_flush(lsn_t lsn) noexcept { ut_ad(!log_sys.is_mmap()); ut_a(log_sys.flush(lsn)); @@ -1150,11 +1157,9 @@ wait and check if an already running write is covering the request. void log_write_up_to(lsn_t lsn, bool durable, const completion_callback *callback) noexcept { - ut_ad(!srv_read_only_mode || log_sys.buf_free_ok()); + ut_ad(!srv_read_only_mode); ut_ad(lsn != LSN_MAX); ut_ad(lsn != 0); - ut_ad(lsn <= log_sys.get_lsn()); - #ifdef HAVE_PMEM if (log_sys.is_mmap()) { @@ -1173,10 +1178,10 @@ void log_write_up_to(lsn_t lsn, bool durable, if (flush_lock.acquire(lsn, callback) != group_commit_lock::ACQUIRED) return; /* Promise to other concurrent flush_lock.acquire() that we - will durable at least up to the current LSN. The LSN may still - advance until we acquire log_sys.latch below. */ - lsn= log_sys.get_lsn(); - flush_lock.set_pending(lsn); + will be durable at least up to the current LSN. The LSN may still + advance when we acquire log_sys.latch below. */ + if (lsn > log_sys.get_flushed_lsn()) + flush_lock.set_pending(lsn); } lsn_t pending_write_lsn= 0, pending_flush_lsn= 0; @@ -1223,33 +1228,41 @@ void log_t::writer_update(bool resizing) noexcept /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable) +void log_buffer_flush_to_disk(bool durable) noexcept { - log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), durable); + log_write_up_to(log_get_lsn(), durable); } /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare() +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept { #ifdef HAVE_PMEM if (log_sys.is_mmap()) return; #endif - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } -void log_t::clear_mmap() +void log_t::clear_mmap() noexcept { - if (!is_mmap() || + if (!is_mmap() || high_level_read_only) + return; #ifdef HAVE_PMEM - !is_opened() || -#endif - high_level_read_only) + if (!is_opened()) + { + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!resize_in_progress()); + ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); + buf_size= unsigned(std::min(capacity(), buf_size_max)); + latch.wr_unlock(); return; + } +#endif + log_resize_acquire(); ut_ad(!resize_in_progress()); ut_ad(write_lsn == get_lsn()); @@ -1259,10 +1272,10 @@ void log_t::clear_mmap() { alignas(16) byte log_block[4096]; const size_t bs{write_size}; - const size_t bf{buf_free.load(std::memory_order_relaxed)}; { - byte *const b= buf; - memcpy_aligned<16>(log_block, b + (bf & ~(bs - 1)), bs); + const size_t bf= + size_t(write_lsn - base_lsn.load(std::memory_order_relaxed)); + memcpy_aligned<16>(log_block, buf + (bf & ~(bs - 1)), bs); } close_file(false); @@ -1270,14 +1283,13 @@ void log_t::clear_mmap() ut_a(attach(log, file_size)); ut_ad(!is_mmap()); - set_buf_free(bf & (bs - 1)); - memcpy_aligned<16>(log_sys.buf, log_block, bs); + memcpy_aligned<16>(buf, log_block, bs); } log_resize_release(); } /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush() +ATTRIBUTE_COLD void log_write_and_flush() noexcept { ut_ad(!srv_read_only_mode); #ifdef HAVE_PMEM @@ -1297,17 +1309,17 @@ Tries to establish a big enough margin of free space in the log, such that a new log entry can be catenated without an immediate need for a checkpoint. NOTE: this function may only be called if the calling thread owns no synchronization objects! */ -ATTRIBUTE_COLD static void log_checkpoint_margin() +ATTRIBUTE_COLD static void log_checkpoint_margin() noexcept { while (log_sys.check_for_checkpoint()) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); ut_ad(!recv_no_log_write); if (!log_sys.check_for_checkpoint()) { func_exit: - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); return; } @@ -1325,7 +1337,7 @@ ATTRIBUTE_COLD static void log_checkpoint_margin() } DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); /* We must wait to prevent the tail of the log overwriting the head. */ buf_flush_wait_flushed(std::min(sync_lsn, checkpoint + (1U << 20))); @@ -1337,7 +1349,7 @@ ATTRIBUTE_COLD static void log_checkpoint_margin() /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check() +void log_free_check() noexcept { ut_ad(!lock_sys.is_holder()); if (log_sys.check_for_checkpoint()) @@ -1354,7 +1366,7 @@ inline void buf_mem_pressure_shutdown() noexcept {} #endif /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept { lsn_t lsn; ulint count = 0; @@ -1482,7 +1494,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn = log_sys.get_lsn(); @@ -1490,7 +1502,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() && lsn != log_sys.last_checkpoint_lsn + sizeof_cp; ut_ad(lsn >= log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); if (lsn_changed) { goto loop; @@ -1508,7 +1520,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() "Free innodb buffer pool"); ut_d(buf_pool.assert_all_freed()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); if (UNIV_UNLIKELY(lsn < recv_sys.lsn)) { @@ -1522,7 +1534,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() /* Make some checks that the server really is quiet */ ut_ad(!srv_any_background_activity()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); } @@ -1533,44 +1545,42 @@ log_print( /*======*/ FILE* file) /*!< in: file where to print */ { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t lsn= log_sys.get_lsn(); mysql_mutex_lock(&buf_pool.flush_list_mutex); const lsn_t pages_flushed = buf_pool.get_oldest_modification(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); + const lsn_t flushed_lsn{log_sys.get_flushed_lsn()}; + const lsn_t checkpoint_lsn{log_sys.last_checkpoint_lsn}; + log_sys.latch.wr_unlock(); fprintf(file, "Log sequence number " LSN_PF "\n" "Log flushed up to " LSN_PF "\n" "Pages flushed up to " LSN_PF "\n" "Last checkpoint at " LSN_PF "\n", - lsn, - log_sys.get_flushed_lsn(), - pages_flushed, - lsn_t{log_sys.last_checkpoint_lsn}); - - log_sys.latch.rd_unlock(); + lsn, flushed_lsn, pages_flushed, checkpoint_lsn); } /** Shut down the redo log subsystem. */ void log_t::close() { ut_ad(this == &log_sys); - ut_ad(!(buf_free & buf_free_LOCK)); if (!is_initialised()) return; close_file(); ut_ad(!checkpoint_buf); ut_ad(!buf); ut_ad(!flush_buf); + base_lsn.store(0, std::memory_order_relaxed); latch.destroy(); - lsn_lock.destroy(); +#ifdef HAVE_PMEM + resize_wrap_mutex.destroy(); +#endif recv_sys.close(); - - max_buf_free= 0; } std::string get_log_file_path(const char *filename) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9047fc9745cd3..214c8eb62c5cc 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1376,6 +1376,7 @@ void recv_sys_t::debug_free() mysql_mutex_lock(&mutex); recovery_on= false; + recv_needed_recovery= false; pages.clear(); pages_it= pages.end(); @@ -1383,7 +1384,6 @@ void recv_sys_t::debug_free() log_sys.clear_mmap(); } - /** Free a redo log snippet. @param data buffer allocated in add() */ inline void recv_sys_t::free(const void *data) @@ -2890,7 +2890,7 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse(source &l, bool if_exists) l - recs + rlen))) { lsn= start_lsn; - if (lsn > log_sys.get_lsn()) + if (lsn > log_sys.get_flushed_lsn(std::memory_order_relaxed)) log_sys.set_recovered_lsn(start_lsn); l+= rlen; offset= begin.ptr - log_sys.buf; @@ -4536,19 +4536,16 @@ dberr_t recv_recovery_read_checkpoint() inline void log_t::set_recovered() noexcept { ut_ad(get_flushed_lsn() == get_lsn()); - ut_ad(recv_sys.lsn == get_lsn()); - size_t offset{recv_sys.offset}; + ut_ad(recv_sys.lsn == get_flushed_lsn()); if (!is_mmap()) { const size_t bs{log_sys.write_size}, bs_1{bs - 1}; - memmove_aligned<512>(buf, buf + (offset & ~bs_1), bs); - offset&= bs_1; + memmove_aligned<512>(buf, buf + (recv_sys.offset & ~bs_1), bs); } -#ifndef _WIN32 +#ifdef HAVE_PMEM else mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); #endif - set_buf_free(offset); } inline bool recv_sys_t::validate_checkpoint() const noexcept @@ -4622,7 +4619,7 @@ dberr_t recv_recovery_from_checkpoint_start() goto err_exit; } ut_ad(recv_sys.file_checkpoint); - ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.scanned_lsn); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; recv_sys.offset = 0; @@ -4684,7 +4681,7 @@ dberr_t recv_recovery_from_checkpoint_start() tablespaces (not individual pages), while retaining the initial recv_sys.pages. */ mysql_mutex_lock(&recv_sys.mutex); - ut_ad(log_sys.get_lsn() >= recv_sys.lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.lsn); recv_sys.clear(); recv_sys.lsn = log_sys.next_checkpoint_lsn; mysql_mutex_unlock(&recv_sys.mutex); @@ -4692,7 +4689,8 @@ dberr_t recv_recovery_from_checkpoint_start() if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { mysql_mutex_lock(&recv_sys.mutex); - deferred_spaces.deferred_dblwr(log_sys.get_lsn()); + deferred_spaces.deferred_dblwr( + log_sys.get_flushed_lsn()); buf_dblwr.recover(); mysql_mutex_unlock(&recv_sys.mutex); } @@ -4725,16 +4723,6 @@ dberr_t recv_recovery_from_checkpoint_start() if (!srv_read_only_mode && log_sys.is_latest()) { log_sys.set_recovered(); - if (recv_needed_recovery - && srv_operation <= SRV_OPERATION_EXPORT_RESTORED - && recv_sys.lsn - log_sys.next_checkpoint_lsn - < log_sys.log_capacity) { - /* Write a FILE_CHECKPOINT marker as the first thing, - before generating any other redo log. This ensures - that subsequent crash recovery will be possible even - if the server were killed soon after this. */ - fil_names_clear(log_sys.next_checkpoint_lsn); - } } DBUG_EXECUTE_IF("before_final_redo_apply", goto err_exit;); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index e9dd66e609152..3b6e925e511f1 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -43,7 +43,6 @@ void (*mtr_t::commit_logger)(mtr_t *, std::pair); #endif std::pair (*mtr_t::finisher)(mtr_t *, size_t); -unsigned mtr_t::spin_wait_delay; void mtr_t::finisher_update() { @@ -52,15 +51,12 @@ void mtr_t::finisher_update() if (log_sys.is_mmap()) { commit_logger= mtr_t::commit_log; - finisher= spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer; + finisher= mtr_t::finish_writer; return; } commit_logger= mtr_t::commit_log; #endif - finisher= - (spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer); + finisher= mtr_t::finish_writer; } void mtr_memo_slot_t::release() const @@ -255,7 +251,7 @@ static void insert_imported(buf_block_t *block) { if (block->page.oldest_modification() <= 1) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); /* For unlogged mtrs (MTR_LOG_NO_REDO), we use the current system LSN. The mtr that generated the LSN is either already committed or in mtr_t::commit. Shared latch and relaxed atomics should be fine here as it is guaranteed @@ -267,7 +263,7 @@ static void insert_imported(buf_block_t *block) mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_pool.insert_into_flush_list (buf_pool.prepare_insert_into_flush_list(lsn), block, lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); } } @@ -337,24 +333,11 @@ void mtr_t::release() m_memo.clear(); } -inline lsn_t log_t::get_write_target() const -{ - ut_ad(latch_have_any()); - if (UNIV_LIKELY(buf_free_ok())) - return 0; - /* The LSN corresponding to the end of buf is - write_lsn - (first_lsn & 4095) + buf_free, - but we use simpler arithmetics to return a smaller write target in - order to minimize waiting in log_write_up_to(). */ - ut_ad(max_buf_free >= 4096 * 4); - return write_lsn + max_buf_free / 2; -} - template void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) + noexcept { size_t modified= 0; - const lsn_t write_lsn= mmap ? 0 : log_sys.get_write_target(); if (mtr->m_made_dirty) { @@ -473,9 +456,6 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) buf_flush_ahead(mtr->m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); - - if (!mmap && UNIV_UNLIKELY(write_lsn != 0)) - log_write_up_to(write_lsn, false); } /** Commit a mini-transaction. */ @@ -692,7 +672,7 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -776,7 +756,7 @@ ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -898,181 +878,110 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) ? ". Shutdown is in progress" : ""); } -static ATTRIBUTE_NOINLINE void lsn_delay(size_t delay, size_t mult) noexcept -{ - delay*= mult * 2; // GCC 13.2.0 -O2 targeting AMD64 wants to unroll twice - HMT_low(); - do - MY_RELAX_CPU(); - while (--delay); - HMT_medium(); -} - -#if defined __clang_major__ && __clang_major__ < 10 -/* Only clang-10 introduced support for asm goto */ -#elif defined __APPLE__ -/* At least some versions of Apple Xcode do not support asm goto */ -#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - __asm__ goto("lock btsq $63, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# else -# define LOCK_TSET \ - __asm__ goto("lock btsl $31, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# endif -#elif defined _MSC_VER && (defined _M_IX86 || defined _M_X64) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - if (!_interlockedbittestandset64 \ - (reinterpret_cast(&buf_free), 63)) return -# else -# define LOCK_TSET \ - if (!_interlockedbittestandset \ - (reinterpret_cast(&buf_free), 31)) return -# endif -#endif - -#ifdef LOCK_TSET -ATTRIBUTE_NOINLINE -void log_t::lsn_lock_bts() noexcept +ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept { - LOCK_TSET; + if (UNIV_LIKELY(!ex)) { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1;; - lsn_delay(delay_iterations, m)) + latch.rd_unlock(); + if (!late) { - if (!(buf_free.load(std::memory_order_relaxed) & buf_free_LOCK)) - LOCK_TSET; - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; + /* Wait for all threads to back off. */ + latch.wr_lock(SRW_LOCK_CALL); + goto got_ex; } - } -# ifdef __GNUC__ - got: - return; -# endif -} + const auto delay= my_cpu_relax_multiplier / 4 * srv_spin_wait_delay; + const auto rounds= srv_n_spin_wait_rounds; -inline -#else -ATTRIBUTE_NOINLINE -#endif -size_t log_t::lock_lsn() noexcept -{ -#ifdef LOCK_TSET - lsn_lock_bts(); - return ~buf_free_LOCK & buf_free.load(std::memory_order_relaxed); -# undef LOCK_TSET -#else - size_t b= buf_free.fetch_or(buf_free_LOCK, std::memory_order_acquire); - if (b & buf_free_LOCK) - { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1; - ((b= buf_free.load(std::memory_order_relaxed)) & buf_free_LOCK) || - (buf_free_LOCK & (b= buf_free.fetch_or(buf_free_LOCK, - std::memory_order_acquire))); - lsn_delay(delay_iterations, m)) - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; + for (;;) + { + HMT_low(); + for (auto r= rounds + 1; r--; ) + { + if (write_lsn_offset.load(std::memory_order_relaxed) & WRITE_BACKOFF) + { + for (auto d= delay; d--; ) + MY_RELAX_CPU(); + } + else + { + HMT_medium(); + goto done; + } + } + HMT_medium(); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } } - return b; -#endif -} - -template -ATTRIBUTE_COLD size_t log_t::append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept -{ - waits++; - ut_ad(buf_free.load(std::memory_order_relaxed) == - (spin ? (b | buf_free_LOCK) : b)); - if (spin) - buf_free.store(b, std::memory_order_release); else - lsn_lock.wr_unlock(); - - if (ex) + { + got_ex: + const uint64_t l= write_lsn_offset.load(std::memory_order_relaxed); + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + ut_d(lsn_t ll= lsn + (l & (WRITE_BACKOFF - 1))); + ut_ad(is_mmap() + ? ll - get_flushed_lsn(std::memory_order_relaxed) < capacity() + : ll - write_lsn - ((write_size - 1) & (write_lsn - first_lsn)) < + buf_size); + waits++; +#ifdef HAVE_PMEM + const bool is_pmem{is_mmap()}; + if (is_pmem) + persist(lsn + (l & (WRITE_BACKOFF - 1))); +#endif latch.wr_unlock(); - else - latch.rd_unlock(); - - log_write_up_to(lsn, is_mmap()); - - if (ex) - latch.wr_lock(SRW_LOCK_CALL); - else - latch.rd_lock(SRW_LOCK_CALL); - - if (spin) - return lock_lsn(); + /* write_buf() or persist() will clear the WRITE_BACKOFF flag, + which our caller will recheck. */ +#ifdef HAVE_PMEM + if (!is_pmem) +#endif + log_write_up_to(lsn + (l & (WRITE_BACKOFF - 1)), false); + if (ex) + { + latch.wr_lock(SRW_LOCK_CALL); + return; + } + } - lsn_lock.wr_lock(); - return buf_free.load(std::memory_order_relaxed); +done: + latch.rd_lock(SRW_LOCK_CALL); } /** Reserve space in the log buffer for appending data. -@tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ -template +template inline std::pair log_t::append_prepare(size_t size, bool ex) noexcept { ut_ad(ex ? latch_have_wr() : latch_have_rd()); ut_ad(mmap == is_mmap()); - if (!spin) - lsn_lock.wr_lock(); - size_t b{spin ? lock_lsn() : buf_free.load(std::memory_order_relaxed)}; - write_to_buf++; - - lsn_t l{lsn.load(std::memory_order_relaxed)}, end_lsn{l + size}; - - if (UNIV_UNLIKELY(mmap - ? (end_lsn - - get_flushed_lsn(std::memory_order_relaxed)) > capacity() - : b + size >= buf_size)) + ut_ad(!mmap || buf_size == std::min(capacity(), buf_size_max)); + const size_t buf_size{this->buf_size - size}; + uint64_t l; + static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); + while (UNIV_UNLIKELY((l= write_lsn_offset.fetch_add(size + WRITE_TO_BUF) & + (WRITE_TO_BUF - 1)) >= buf_size)) { - b= append_prepare_wait(b, ex, l); - /* While flushing log, we had released the lsn lock and LSN could have - progressed in the meantime. */ - l= lsn.load(std::memory_order_relaxed); - end_lsn= l + size; + /* The following is inlined here instead of being part of + append_prepare_wait(), in order to increase the locality of reference + and to set the WRITE_BACKOFF flag as soon as possible. */ + bool late(write_lsn_offset.fetch_or(WRITE_BACKOFF) & WRITE_BACKOFF); + /* Subtract our LSN overshoot. */ + write_lsn_offset.fetch_sub(size); + append_prepare_wait(late, ex); } - size_t new_buf_free= b + size; - if (mmap && new_buf_free >= file_size) - new_buf_free-= size_t(capacity()); - - lsn.store(end_lsn, std::memory_order_relaxed); + const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}, + end_lsn{lsn + size}; if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) set_check_for_checkpoint(true); - byte *our_buf= buf; - if (spin) - buf_free.store(new_buf_free, std::memory_order_release); - else - { - buf_free.store(new_buf_free, std::memory_order_relaxed); - lsn_lock.wr_unlock(); - } - - return {l, our_buf + b}; + return {lsn, + buf + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; } /** Finish appending data to the log. @@ -1217,7 +1126,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (!resize_flush_buf) { ut_ad(is_mmap()); - lsn_lock.wr_lock(); + resize_wrap_mutex.wr_lock(); const size_t resize_capacity{resize_target - START_OFFSET}; { const lsn_t resizing{resize_in_progress()}; @@ -1228,7 +1137,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (UNIV_UNLIKELY(lsn < resizing)) { /* This function may execute in multiple concurrent threads - that hold a shared log_sys.latch. Before we got lsn_lock, + that hold a shared log_sys.latch. Before we got resize_wrap_mutex, another thread could have executed resize_lsn.store(lsn) below with a larger lsn than ours. @@ -1278,7 +1187,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, ut_ad(resize_buf[s] <= 1); resize_buf[s]= 1; mmap_done: - lsn_lock.wr_unlock(); + resize_wrap_mutex.wr_unlock(); } else #endif @@ -1305,7 +1214,7 @@ inline void log_t::append(byte *&d, const void *s, size_t size) noexcept d+= size; } -template +template std::pair mtr_t::finish_writer(mtr_t *mtr, size_t len) { @@ -1316,7 +1225,7 @@ mtr_t::finish_writer(mtr_t *mtr, size_t len) const size_t size{mtr->m_commit_lsn ? 5U + 8U : 5U}; std::pair start= - log_sys.append_prepare(len, mtr->m_latch_ex); + log_sys.append_prepare(len, mtr->m_latch_ex); if (!mmap) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 73420261b1a21..39d48aafda21c 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -68,7 +68,7 @@ Created 9/17/2000 Heikki Tuuri /** Delay an INSERT, DELETE or UPDATE operation if the purge is lagging. */ -static void row_mysql_delay_if_needed() +static void row_mysql_delay_if_needed() noexcept { const auto delay= srv_dml_needed_delay; if (UNIV_UNLIKELY(delay != 0)) @@ -77,8 +77,8 @@ static void row_mysql_delay_if_needed() log_sys.latch.rd_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; + const lsn_t lsn= log_sys.get_flushed_lsn(); log_sys.latch.rd_unlock(); - const lsn_t lsn= log_sys.get_lsn(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index b1450f93321d8..b72ee055b671b 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -1376,7 +1376,7 @@ srv_mon_process_existing_counter( /* innodb_os_log_written */ case MONITOR_OVLD_OS_LOG_WRITTEN: - value = log_sys.get_lsn() - recv_sys.lsn; + value = log_get_lsn() - recv_sys.lsn; break; /* innodb_log_waits */ @@ -1477,7 +1477,7 @@ srv_mon_process_existing_counter( break; case MONITOR_OVLD_LSN_CURRENT: - value = log_sys.get_lsn(); + value = log_get_lsn(); break; case MONITOR_OVLD_CHECKPOINTS: @@ -1485,10 +1485,10 @@ srv_mon_process_existing_counter( break; case MONITOR_LSN_CHECKPOINT_AGE: - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); value = static_cast(log_sys.get_lsn() - log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); break; case MONITOR_OVLD_BUF_OLDEST_LSN: diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 561b8610ff369..f8c48d0c91f7c 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -948,13 +948,13 @@ srv_export_innodb_status(void) mysql_mutex_unlock(&srv_innodb_monitor_mutex); - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); export_vars.innodb_lsn_current = log_sys.get_lsn(); export_vars.innodb_lsn_flushed = log_sys.get_flushed_lsn(); export_vars.innodb_lsn_last_checkpoint = log_sys.last_checkpoint_lsn; export_vars.innodb_checkpoint_max_age = static_cast( log_sys.max_checkpoint_age); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); export_vars.innodb_os_log_written = export_vars.innodb_lsn_current - recv_sys.lsn; @@ -1041,7 +1041,7 @@ void srv_monitor_task(void*) /* Try to track a strange bug reported by Harald Fuchs and others, where the lsn seems to decrease at times */ - lsn_t new_lsn = log_sys.get_lsn(); + lsn_t new_lsn = log_get_lsn(); ut_a(new_lsn >= old_lsn); old_lsn = new_lsn; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 4ca8c67743ded..4d2f72196bc0c 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1077,7 +1077,7 @@ srv_init_abort_low( /** Prepare to delete the redo log file. Flush the dirty pages from all the buffer pools. Flush the redo log buffer to the redo log file. @return lsn upto which data pages have been flushed. */ -ATTRIBUTE_COLD static lsn_t srv_prepare_to_delete_redo_log_file() +ATTRIBUTE_COLD static lsn_t srv_prepare_to_delete_redo_log_file() noexcept { DBUG_ENTER("srv_prepare_to_delete_redo_log_file"); @@ -1091,7 +1091,7 @@ ATTRIBUTE_COLD static lsn_t srv_prepare_to_delete_redo_log_file() log_sys.latch.wr_lock(SRW_LOCK_CALL); const bool latest_format{log_sys.is_latest()}; - lsn_t flushed_lsn{log_sys.get_lsn()}; + lsn_t flushed_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; if (latest_format && !(log_sys.file_size & 4095) && flushed_lsn != log_sys.next_checkpoint_lsn + @@ -1099,6 +1099,11 @@ ATTRIBUTE_COLD static lsn_t srv_prepare_to_delete_redo_log_file() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT)) { +#ifdef HAVE_PMEM + if (!log_sys.is_opened()) + log_sys.buf_size= unsigned(std::min(log_sys.capacity(), + log_sys.buf_size_max)); +#endif fil_names_clear(flushed_lsn); flushed_lsn= log_sys.get_lsn(); } @@ -1139,7 +1144,7 @@ ATTRIBUTE_COLD static lsn_t srv_prepare_to_delete_redo_log_file() if (latest_format) log_write_up_to(flushed_lsn, false); - ut_ad(flushed_lsn == log_sys.get_lsn()); + ut_ad(flushed_lsn == log_get_lsn()); ut_ad(!os_aio_pending_reads()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); @@ -1223,6 +1228,18 @@ ATTRIBUTE_COLD static dberr_t ibuf_log_rebuild_if_needed() return err; } +inline lsn_t log_t::init_lsn() noexcept +{ + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!write_lsn_offset); + write_lsn_offset= 0; + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; + latch.wr_unlock(); + return lsn; +} + /** Start InnoDB. @param[in] create_new_db whether to create a new database @return DB_SUCCESS or error code */ From 47521545137a8889935591358880006849549389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 8 May 2025 11:18:16 +0300 Subject: [PATCH 2/3] MDEV-36760 log_t::append_prepare_wait(): Bogus assertion on write_lsn log_t::append_prepare_wait(): Do not attempt to read log_sys.write_lsn because it is not protected by log_sys.latch but by write_lock, which we cannot hold here. The assertion could fail if log_t::write_buf() is executing concurrently, and it has not yet executed log_write_buf() or updated log_sys.write_lsn. Fixes up commit acd071f599f416ddb4821dec485c4d912844213f (MDEV-21923) (cherry picked from commit 84dd2437c507ed194da03fe04fea14e261e47bc5) --- storage/innobase/mtr/mtr0mtr.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 3b6e925e511f1..6d67cfa6b0734 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -917,17 +917,16 @@ ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept { got_ex: const uint64_t l= write_lsn_offset.load(std::memory_order_relaxed); - const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; - ut_d(lsn_t ll= lsn + (l & (WRITE_BACKOFF - 1))); - ut_ad(is_mmap() - ? ll - get_flushed_lsn(std::memory_order_relaxed) < capacity() - : ll - write_lsn - ((write_size - 1) & (write_lsn - first_lsn)) < - buf_size); + const lsn_t lsn= base_lsn.load(std::memory_order_relaxed) + + (l & (WRITE_BACKOFF - 1)); waits++; #ifdef HAVE_PMEM const bool is_pmem{is_mmap()}; if (is_pmem) - persist(lsn + (l & (WRITE_BACKOFF - 1))); + { + ut_ad(lsn - get_flushed_lsn(std::memory_order_relaxed) < capacity()); + persist(lsn); + } #endif latch.wr_unlock(); /* write_buf() or persist() will clear the WRITE_BACKOFF flag, @@ -935,7 +934,7 @@ ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept #ifdef HAVE_PMEM if (!is_pmem) #endif - log_write_up_to(lsn + (l & (WRITE_BACKOFF - 1)), false); + log_write_up_to(lsn, false); if (ex) { latch.wr_lock(SRW_LOCK_CALL); From 8f486f2f74a81d568b0ae1c89f7971cbff4ed6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 May 2025 14:14:20 +0300 Subject: [PATCH 3/3] MDEV-36301 SET GLOBAL innodb_log_file_disabled, ... innodb_log_file_disabled: A new Boolean settable global parameter, for disabling the InnoDB redo log. When set, the server is not crash safe. innodb_log_group_home_dir: Allow the value to be changed with SET GLOBAL, as long as the log remains in the same file system or innodb_log_file_disabled=ON was set. innodb_log_update(): A common function for implementing SET GLOBAL innodb_log_file_size, innodb_log_file_disabled, innodb_log_checkpoint_now, innodb_log_group_home_dir. log_sys.buf_size_requested: The configured innodb_log_buffer_size. While the log is disabled, we may set log_sys.buf_size (the actual size of log_sys.buf) differently. log_sys.disabled: The current setting of innodb_log_file_disabled. log_t::append_prepare(): Instead of referring to file_size or capacity() for mmap=true, always refer to buf_size. When log_sys.disabled holds, log_sys.buf may be much smaller than log_sys.file_size. Its size is always reflected by log_sys.buf_size. log_t::attach(): Handle log_sys.disabled. log_t::disable(): Implements SET innodb_log_file_disabled=ON. Even if the log used to be in persistent memory, here we will set up dummy log_sys.buf and log_sys.flush_buf so that the dummy writes will appear to use file based writes. log_t::skip_write_buf(): A dummy log writer implementation that is used when log_sys.disabled holds and the log is not being resized. log_t::resize_abort(): When the log remains disabled, "persist" all the log and advance the group_lock and flush_lock to the current LSN, just like log_t::disable() does. log_t::resize_start(): Handle log_sys.disabled, that is, SET GLOBAL innodb_log_file_disabled=OFF when the redo log had previously been disabled. If we are on persistent memory, we will "fake" the dummy log_sys.buf to appear as memory-mapped as well, so that log_t::resize_write() and log_t::resize_write_buf() can assume that both log files are of the same type (memory-mapped or file-based). The dummy log_sys.flush_buf will be stored in log_sys.resize_flush_buf in that case. When moving from memory-mapped to a regular log file, we will allocate log_sys.checkpoint_buf. It will be freed in log_t::close(). log_t::resize_rename(): When innodb_log_group_home_dir is changed between file systems, handle the non-atomic replacement of the log file in a special way. For a moment, a recoverable ib_logfile0 will exist in both locations. log_t::resize_write(): Detect memory-mapped log by !resize_log.is_opened(). If the memory-mapped log is being re-enabled, log_sys.resize_flush_buf may temporarily store the value of a dummy log_sys.flush_buf. During any log resizing or disabling or enabling, both buf and resize_buf must appear to be either file-based or memory-mapped. log_t::write_buf(), log_write_up_to(): Handle the special case that disable() executed or resize_start() started re-enabling the log while we were waiting for flush_lock or write_lock. In the latter case, log_sys.writer will be changed from log_t::skip_write_buf to log_writer_resizing during the execution of log_write_up_to(). log_t::persist(): Skip the writes if the log is disabled, that is, a memory-mapped log is in the process being re-enabled. In this case, we cannot trust log_sys.file_size, and the pmem_persist() could be invoked on an invalid address range. log_resize_acquire_group(): Acquier the group locks for write and flush. log_resize_release_group(): Release the group locks for write and flush. log_resize_acquire(): Return whether the group locks were elided. log_write_and_flush_prepare(), log_write_and_flush(): Protect also the log_sys.is_mmap() case with write_lock and flush_lock, in order to prevent a race condition between mtr_t::commit_file() and log_t::disable(). log_t::persist(): Remove the assertions about not holding write_lock or flush_lock. We will hold them during DDL operations. --- extra/mariabackup/xtrabackup.cc | 4 +- .../suite/innodb/r/log_file_disabled.result | 30 + .../suite/innodb/r/log_file_overwrite.result | 5 +- .../suite/innodb/r/log_file_size.result | 2 +- .../innodb/r/log_file_size_online.result | 28 +- .../suite/innodb/t/log_file_disabled.opt | 1 + .../suite/innodb/t/log_file_disabled.test | 45 ++ .../suite/innodb/t/log_file_overwrite.test | 2 + mysql-test/suite/innodb/t/log_file_size.test | 2 +- .../suite/innodb/t/log_file_size_online.test | 28 +- .../r/innodb_log_group_home_dir_basic.result | 9 +- .../suite/sys_vars/r/sysvars_innodb.result | 14 +- .../t/innodb_log_group_home_dir_basic.test | 8 +- storage/innobase/buf/buf0flu.cc | 79 ++- storage/innobase/handler/ha_innodb.cc | 300 ++++++---- storage/innobase/include/log0log.h | 52 +- storage/innobase/log/log0log.cc | 537 ++++++++++++++---- storage/innobase/mtr/mtr0mtr.cc | 16 +- storage/innobase/srv/srv0start.cc | 69 ++- 19 files changed, 948 insertions(+), 283 deletions(-) create mode 100644 mysql-test/suite/innodb/r/log_file_disabled.result create mode 100644 mysql-test/suite/innodb/t/log_file_disabled.opt create mode 100644 mysql-test/suite/innodb/t/log_file_disabled.test diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 1af5cdf87baa6..c696a34e0e6a2 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1895,8 +1895,8 @@ struct my_option xb_server_options[] = {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "Redo log buffer size in bytes.", - (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0, - GET_UINT, REQUIRED_ARG, 2U << 20, + (G_PTR*) &log_sys.buf_size_requested, (G_PTR*) &log_sys.buf_size_requested, + 0, GET_UINT, REQUIRED_ARG, 2U << 20, 2U << 20, log_sys.buf_size_max, 0, 4096, 0}, {"innodb_log_file_mmap", OPT_INNODB_LOG_FILE_SIZE, "Whether ib_logfile0 should be memory-mapped", diff --git a/mysql-test/suite/innodb/r/log_file_disabled.result b/mysql-test/suite/innodb/r/log_file_disabled.result new file mode 100644 index 0000000000000..a4e7a3359ba10 --- /dev/null +++ b/mysql-test/suite/innodb/r/log_file_disabled.result @@ -0,0 +1,30 @@ +# restart: --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-undo-tablespaces=0 --innodb-log-file-disabled +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +InnoDB YES Supports transactions, row-level locking, foreign keys and encryption for tables YES YES YES +# restart: --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-undo-tablespaces=0 +SET GLOBAL innodb_log_file_disabled=ON; +SET STATEMENT max_statement_time=1e-6 FOR +SET GLOBAL innodb_log_file_disabled=OFF; +SET GLOBAL innodb_log_file_disabled=OFF; +CREATE TABLE t(a SERIAL, b CHAR(255) NOT NULL DEFAULT '') ENGINE=InnoDB; +SET GLOBAL innodb_log_file_disabled=ON; +ALTER TABLE t ADD c2 CHAR FIRST; +INSERT INTO t(a) SELECT * from seq_1_to_10000; +SET STATEMENT max_statement_time=1e-6 FOR +SET GLOBAL innodb_log_file_disabled=OFF; +DROP TABLE t; +SET GLOBAL innodb_log_file_disabled=OFF; +SET GLOBAL innodb_log_file_disabled=ON, +innodb_log_file_size=@@innodb_log_file_size + 4096; +CREATE TABLE t ENGINE=InnoDB SELECT 1; +SET GLOBAL innodb_log_file_disabled=OFF, +innodb_log_file_size=@@innodb_log_file_size - 4096; +# restart: --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log --innodb-undo-tablespaces=0 +SELECT * FROM t; +1 +1 +DROP TABLE t; +# restart diff --git a/mysql-test/suite/innodb/r/log_file_overwrite.result b/mysql-test/suite/innodb/r/log_file_overwrite.result index ddf8cef63e7c8..7fb378c1f2072 100644 --- a/mysql-test/suite/innodb/r/log_file_overwrite.result +++ b/mysql-test/suite/innodb/r/log_file_overwrite.result @@ -12,11 +12,8 @@ INSERT INTO t1 SELECT seq, repeat('a', 4000) FROM seq_1_to_1800; # # restart: --innodb-force-recovery=6 SET GLOBAL innodb_log_checkpoint_now=1; -Warnings: -Warning 138 InnoDB doesn't force checkpoint when innodb-force-recovery=6. # restart: --innodb-read-only=1 SET GLOBAL innodb_log_checkpoint_now=1; -Warnings: -Warning 138 InnoDB doesn't force checkpoint when innodb-read-only=1. # restart +SET GLOBAL innodb_log_checkpoint_now=1; DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/log_file_size.result b/mysql-test/suite/innodb/r/log_file_size.result index 71fe06cf23bc4..54994fd597523 100644 --- a/mysql-test/suite/innodb/r/log_file_size.result +++ b/mysql-test/suite/innodb/r/log_file_size.result @@ -30,7 +30,7 @@ connection default; # restart: --innodb-log-group-home-dir=foo\;bar SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 1 /syntax error in innodb_log_group_home_dir/ in mysqld.1.err +FOUND 1 /InnoDB: File foo.*bar/ib_logfile0 was not found/ in mysqld.1.err # restart: --debug-dbug=d,innodb_log_abort_1 SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' diff --git a/mysql-test/suite/innodb/r/log_file_size_online.result b/mysql-test/suite/innodb/r/log_file_size_online.result index 8dcd9a47b2f0b..487f571e3ac06 100644 --- a/mysql-test/suite/innodb/r/log_file_size_online.result +++ b/mysql-test/suite/innodb/r/log_file_size_online.result @@ -61,16 +61,38 @@ SELECT COUNT(*),LENGTH(b) FROM t GROUP BY b; COUNT(*) LENGTH(b) 9 0 19991 255 +SELECT @@innodb_log_group_home_dir; +@@innodb_log_group_home_dir +./ +ib_logfile0 SHOW VARIABLES LIKE 'innodb_log_file_size'; Variable_name Value innodb_log_file_size 5242880 -SET GLOBAL innodb_log_file_size=6291456; +SET GLOBAL innodb_log_file_disabled=ON; +SET GLOBAL innodb_log_group_home_dir='log'; +SELECT @@innodb_log_group_home_dir; +@@innodb_log_group_home_dir +log +SET GLOBAL innodb_log_file_size=6291456,innodb_log_file_disabled=OFF, +innodb_log_group_home_dir='log/'; +ib_logfile0 SHOW VARIABLES LIKE 'innodb_log_file_size'; Variable_name Value innodb_log_file_size 6291456 -SET GLOBAL innodb_log_file_size=5242880; +SELECT @@innodb_log_group_home_dir; +@@innodb_log_group_home_dir +log/ +SET GLOBAL innodb_log_group_home_dir='log'; +ib_logfile0 +SELECT @@innodb_log_group_home_dir; +@@innodb_log_group_home_dir +log +SET GLOBAL innodb_log_file_size=5242880,innodb_log_group_home_dir=default; SHOW VARIABLES LIKE 'innodb_log_file_size'; Variable_name Value innodb_log_file_size 5242880 -FOUND 1 /InnoDB: Resized log to 6\.000MiB/ in mysqld.1.err +SHOW VARIABLES LIKE 'innodb_group_home_dir'; +Variable_name Value +ib_logfile0 +FOUND 3 /InnoDB: Resized log to 6\.000MiB/ in mysqld.1.err DROP TABLE t; diff --git a/mysql-test/suite/innodb/t/log_file_disabled.opt b/mysql-test/suite/innodb/t/log_file_disabled.opt new file mode 100644 index 0000000000000..338d491625a00 --- /dev/null +++ b/mysql-test/suite/innodb/t/log_file_disabled.opt @@ -0,0 +1 @@ +--skip-innodb-undo-tablespaces diff --git a/mysql-test/suite/innodb/t/log_file_disabled.test b/mysql-test/suite/innodb/t/log_file_disabled.test new file mode 100644 index 0000000000000..f7d11c831df12 --- /dev/null +++ b/mysql-test/suite/innodb/t/log_file_disabled.test @@ -0,0 +1,45 @@ +--source include/have_innodb.inc +--source include/have_sequence.inc + +let $bugdir= $MYSQLTEST_VARDIR/tmp/log; +--mkdir $bugdir +let $check_no_innodb=SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); + +--let $ibp=--innodb-log-group-home-dir=$bugdir --innodb-data-home-dir=$bugdir +--let $ibp=$ibp --innodb-undo-tablespaces=0 + +--let $restart_parameters= $ibp --innodb-log-file-disabled +--source include/restart_mysqld.inc +eval $check_no_innodb; +--rmdir $bugdir +--mkdir $bugdir +--let $restart_parameters= $ibp +--source include/restart_mysqld.inc +SET GLOBAL innodb_log_file_disabled=ON; +SET STATEMENT max_statement_time=1e-6 FOR +SET GLOBAL innodb_log_file_disabled=OFF; +SET GLOBAL innodb_log_file_disabled=OFF; +CREATE TABLE t(a SERIAL, b CHAR(255) NOT NULL DEFAULT '') ENGINE=InnoDB; +SET GLOBAL innodb_log_file_disabled=ON; +ALTER TABLE t ADD c2 CHAR FIRST; +INSERT INTO t(a) SELECT * from seq_1_to_10000; +SET STATEMENT max_statement_time=1e-6 FOR +SET GLOBAL innodb_log_file_disabled=OFF; +DROP TABLE t; +SET GLOBAL innodb_log_file_disabled=OFF; + +#--error ER_WRONG_VALUE_FOR_VAR +#SET GLOBAL innodb_log_file_disabled=ON,innodb_log_file_size=10485761; +SET GLOBAL innodb_log_file_disabled=ON, +innodb_log_file_size=@@innodb_log_file_size + 4096; +CREATE TABLE t ENGINE=InnoDB SELECT 1; +SET GLOBAL innodb_log_file_disabled=OFF, +innodb_log_file_size=@@innodb_log_file_size - 4096; +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $bugdir diff --git a/mysql-test/suite/innodb/t/log_file_overwrite.test b/mysql-test/suite/innodb/t/log_file_overwrite.test index 253cf2d611925..4a0c1efe22a5d 100644 --- a/mysql-test/suite/innodb/t/log_file_overwrite.test +++ b/mysql-test/suite/innodb/t/log_file_overwrite.test @@ -29,4 +29,6 @@ SET GLOBAL innodb_log_checkpoint_now=1; let $restart_parameters=; --source include/restart_mysqld.inc +SET GLOBAL innodb_log_checkpoint_now=1; + DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test index 01fcb84860a2f..d3798f5d3f855 100644 --- a/mysql-test/suite/innodb/t/log_file_size.test +++ b/mysql-test/suite/innodb/t/log_file_size.test @@ -105,7 +105,7 @@ DELETE FROM t1 WHERE a=0; --source include/start_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_PATTERN= syntax error in innodb_log_group_home_dir; +--let SEARCH_PATTERN= InnoDB: File foo.*bar/ib_logfile0 was not found --source include/search_pattern_in_file.inc --let $restart_parameters= --debug-dbug=d,innodb_log_abort_1 diff --git a/mysql-test/suite/innodb/t/log_file_size_online.test b/mysql-test/suite/innodb/t/log_file_size_online.test index b511199bd8265..7fb3e1bb75458 100644 --- a/mysql-test/suite/innodb/t/log_file_size_online.test +++ b/mysql-test/suite/innodb/t/log_file_size_online.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_sequence.inc +--source include/not_embedded.inc --source include/no_valgrind_without_big.inc # @@ -76,13 +77,34 @@ reap; SELECT * FROM t WHERE a<10; SELECT COUNT(*),LENGTH(b) FROM t GROUP BY b; - +SELECT @@innodb_log_group_home_dir; +let $datadir=`select @@datadir`; +--list_files $datadir ib_logfile* +--mkdir $datadir/log SHOW VARIABLES LIKE 'innodb_log_file_size'; -SET GLOBAL innodb_log_file_size=6291456; +SET GLOBAL innodb_log_file_disabled=ON; +--list_files $datadir ib_logfile* +--list_files $datadir/log ib_logfile* +SET GLOBAL innodb_log_group_home_dir='log'; +SELECT @@innodb_log_group_home_dir; +--list_files $datadir/log ib_logfile* +SET GLOBAL innodb_log_file_size=6291456,innodb_log_file_disabled=OFF, +innodb_log_group_home_dir='log/'; +--list_files $datadir/log ib_logfile* SHOW VARIABLES LIKE 'innodb_log_file_size'; -SET GLOBAL innodb_log_file_size=5242880; +SELECT @@innodb_log_group_home_dir; +--list_files $datadir ib_logfile* +SET GLOBAL innodb_log_group_home_dir='log'; +--list_files $datadir/log ib_logfile* +SELECT @@innodb_log_group_home_dir; +--list_files $datadir ib_logfile* +SET GLOBAL innodb_log_file_size=5242880,innodb_log_group_home_dir=default; SHOW VARIABLES LIKE 'innodb_log_file_size'; +--list_files $datadir/log ib_logfile* +SHOW VARIABLES LIKE 'innodb_group_home_dir'; +--list_files $datadir ib_logfile* let SEARCH_PATTERN = InnoDB: Resized log to 6\\.000MiB; --source include/search_pattern_in_file.inc +--rmdir $datadir/log DROP TABLE t; diff --git a/mysql-test/suite/sys_vars/r/innodb_log_group_home_dir_basic.result b/mysql-test/suite/sys_vars/r/innodb_log_group_home_dir_basic.result index 35542b4ae7a9b..7f3e7ede601c3 100644 --- a/mysql-test/suite/sys_vars/r/innodb_log_group_home_dir_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_log_group_home_dir_basic.result @@ -5,12 +5,19 @@ COUNT(@@GLOBAL.innodb_log_group_home_dir) 1 Expected '#---------------------BS_STVARS_036_02----------------------#' SET @@GLOBAL.innodb_log_group_home_dir=1; -ERROR HY000: Variable 'innodb_log_group_home_dir' is a read only variable +ERROR 42000: Incorrect argument type to variable 'innodb_log_group_home_dir' Expected error 'Read only variable' SELECT COUNT(@@GLOBAL.innodb_log_group_home_dir); COUNT(@@GLOBAL.innodb_log_group_home_dir) 1 1 Expected +SELECT @@GLOBAL.innodb_log_group_home_dir; +@@GLOBAL.innodb_log_group_home_dir +./ +SET @@GLOBAL.innodb_log_group_home_dir=DEFAULT; +SELECT @@GLOBAL.innodb_log_group_home_dir; +@@GLOBAL.innodb_log_group_home_dir +./ '#---------------------BS_STVARS_036_03----------------------#' SELECT @@GLOBAL.innodb_log_group_home_dir = VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 1043684ba2c34..4327dbc202d26 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -955,6 +955,18 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_LOG_FILE_DISABLED +SESSION_VALUE NULL +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Whether the ib_logfile0 is disabled (and InnoDB is crash-unsafe) +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME INNODB_LOG_FILE_MMAP SESSION_VALUE NULL DEFAULT_VALUE ON @@ -1001,7 +1013,7 @@ NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_SPIN_WAIT_DELAY SESSION_VALUE NULL diff --git a/mysql-test/suite/sys_vars/t/innodb_log_group_home_dir_basic.test b/mysql-test/suite/sys_vars/t/innodb_log_group_home_dir_basic.test index d6d5446c4c7ac..f73beba096a3e 100644 --- a/mysql-test/suite/sys_vars/t/innodb_log_group_home_dir_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_log_group_home_dir_basic.test @@ -1,5 +1,4 @@ - ################## mysql-test\t\innodb_log_group_home_dir_basic.test ########## # # # Variable Name: innodb_log_group_home_dir # @@ -22,6 +21,7 @@ # # ############################################################################### +--source include/not_embedded.inc --source include/have_innodb.inc --echo '#---------------------BS_STVARS_036_01----------------------#' @@ -37,13 +37,15 @@ SELECT COUNT(@@GLOBAL.innodb_log_group_home_dir); # Check if Value can set # #################################################################### ---error ER_INCORRECT_GLOBAL_LOCAL_VAR +--error ER_WRONG_TYPE_FOR_VAR SET @@GLOBAL.innodb_log_group_home_dir=1; --echo Expected error 'Read only variable' SELECT COUNT(@@GLOBAL.innodb_log_group_home_dir); --echo 1 Expected - +SELECT @@GLOBAL.innodb_log_group_home_dir; +SET @@GLOBAL.innodb_log_group_home_dir=DEFAULT; +SELECT @@GLOBAL.innodb_log_group_home_dir; diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 65c47fe4801d6..91244c6034727 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1784,12 +1784,27 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept const size_t offset{(n & 1) ? CHECKPOINT_2 : CHECKPOINT_1}; static_assert(CPU_LEVEL1_DCACHE_LINESIZE >= 64, "efficiency"); static_assert(CPU_LEVEL1_DCACHE_LINESIZE <= 4096, "compatibility"); - byte* c= my_assume_aligned - (is_mmap() ? buf + offset : checkpoint_buf); - memset_aligned(c, 0, CPU_LEVEL1_DCACHE_LINESIZE); - mach_write_to_8(my_assume_aligned<8>(c), next_checkpoint_lsn); - mach_write_to_8(my_assume_aligned<8>(c + 8), end_lsn); - mach_write_to_4(my_assume_aligned<4>(c + 60), my_crc32c(0, c, 60)); + byte* c= my_assume_aligned( +#ifdef HAVE_PMEM + is_mmap() ? buf + offset : +#endif + checkpoint_buf); +#ifdef HAVE_PMEM + if (!c) + { + ut_ad(disabled); + ut_ad(writer == skip_write_buf); + ut_ad(resize_in_progress() <= 1); + } + else +#endif + { + memset_aligned + (c, 0, CPU_LEVEL1_DCACHE_LINESIZE); + mach_write_to_8(my_assume_aligned<8>(c), next_checkpoint_lsn); + mach_write_to_8(my_assume_aligned<8>(c + 8), end_lsn); + mach_write_to_4(my_assume_aligned<4>(c + 60), my_crc32c(0, c, 60)); + } lsn_t resizing; @@ -1819,7 +1834,8 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(ut_is_2pow(write_size)); ut_ad(write_size >= 512); ut_ad(write_size <= 4096); - log.write(offset, {c, write_size}); + if (UNIV_LIKELY(!disabled)) + log.write(offset, {c, write_size}); if (resizing > 1 && resizing <= next_checkpoint_lsn) { resize_log.write(CHECKPOINT_1, {c, write_size}); @@ -1830,7 +1846,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept aligned_free(buf); } - if (!log_write_through) + if (!log_write_through && UNIV_LIKELY(!disabled)) ut_a(log.flush()); latch.wr_lock(SRW_LOCK_CALL); ut_ad(checkpoint_pending); @@ -1858,24 +1874,31 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept if (resizing > 1 && resizing <= checkpoint_lsn) { - ut_ad(is_mmap() == !resize_flush_buf); - ut_ad(is_mmap() == !resize_log.is_opened()); + ut_ad(resize_dir); +#ifdef HAVE_PMEM + ut_ad(is_mmap() == !resize_flush_buf || disabled); - if (!is_mmap()) + if (resize_log.is_opened()) +#endif { + ut_ad(disabled == !log.is_opened()); if (!log_write_through) ut_a(resize_log.flush()); - IF_WIN(log.close(),); + IF_WIN(if (!disabled) log.close(),); } if (resize_rename()) { - /* Resizing failed. Discard the ib_logfile101. */ + /* Resizing failed. @see resize_abort() */ #ifdef HAVE_PMEM - if (is_mmap()) + if (!resize_log.is_opened()) { ut_ad(!is_opened()); my_munmap(resize_buf, resize_target); + flush_buf= resize_flush_buf; + resize_flush_buf= nullptr; + if (flush_buf) + writer= skip_write_buf; } else #endif @@ -1883,6 +1906,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(!is_mmap()); ut_free_dodump(resize_buf, buf_size); ut_free_dodump(resize_flush_buf, buf_size); + resize_log.close(); #ifdef _WIN32 ut_ad(!log.is_opened()); bool success; @@ -1898,10 +1922,17 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept { /* Adopt the resized log. */ #ifdef HAVE_PMEM - if (is_mmap()) + if (!resize_log.is_opened()) { ut_ad(!is_opened()); - my_munmap(buf, file_size); + if (disabled) + { + ut_free_dodump(buf, buf_size); + /* log_t::resize_start() stored the dummy flush_buf here. */ + ut_free_dodump(resize_flush_buf, buf_size); + } + else + my_munmap(buf, file_size); buf= resize_buf; buf_size= unsigned(std::min(resize_target - START_OFFSET, buf_size_max)); @@ -1909,8 +1940,7 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept else #endif { - ut_ad(!is_mmap()); - IF_WIN(,log.close()); + IF_WIN(,if (!disabled) log.close()); std::swap(log, resize_log); ut_free_dodump(buf, buf_size); ut_free_dodump(flush_buf, buf_size); @@ -1920,14 +1950,25 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept srv_log_file_size= resizing_completed= file_size= resize_target; first_lsn= resizing; set_capacity(); + disabled= false; } ut_ad(!resize_log.is_opened()); resize_buf= nullptr; resize_flush_buf= nullptr; resize_target= 0; resize_lsn.store(0, std::memory_order_relaxed); + if (srv_log_group_home_dir != resize_dir) + { + my_free(srv_log_group_home_dir); + srv_log_group_home_dir= my_strdup(PSI_NOT_INSTRUMENTED, resize_dir, + MYF(MY_FAE)); + } + resize_dir= nullptr; resize_initiator= nullptr; - writer_update(false); + if (UNIV_UNLIKELY(disabled)) + writer= skip_write_buf; + else + writer_update(false); } log_resize_release(); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 41c564776f751..085bee6b0c38a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -222,6 +222,24 @@ enum default_row_format_enum { static my_bool innodb_truncate_temporary_tablespace_now; +/** Group checking of system variables */ +namespace group +{ + /** the specified parameters */ + enum spec + { + NONE= 0, + /** innodb_log_file_disabled is specified */ + LOG_DISABLED= 1<<0, + /** innodb_log_file_size is specified */ + LOG_SIZE= 1<<1, + /** innodb_log_checkpoint_now is specified */ + LOG_CHECKPOINT= 1<<2, + /** innodb_log_group_home_dir is specified */ + LOG_DIR= 1<<3 + }; +} + /** Whether ROW_FORMAT=COMPRESSED tables are read-only */ static my_bool innodb_read_only_compressed; @@ -3679,6 +3697,7 @@ static int innodb_init_abort() srv_tmp_space.delete_files(); } srv_tmp_space.shutdown(); + my_free(srv_log_group_home_dir); DBUG_RETURN(1); } @@ -3803,6 +3822,8 @@ static int innodb_init_params() if (!srv_page_size_shift) { sql_print_error("InnoDB: Invalid page size=%lu.\n", srv_page_size); + err: + srv_log_group_home_dir= nullptr; DBUG_RETURN(HA_ERR_INITIALIZATION); } @@ -3832,7 +3853,7 @@ static int innodb_init_params() sql_print_error("InnoDB: innodb_page_size=%lu requires " "innodb_buffer_pool_size >= %zu MiB current %zu MiB", srv_page_size, min >> 20, innodb_buffer_pool_size >> 20); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } if (!ut_is_2pow(log_sys.write_size)) @@ -3840,12 +3861,12 @@ static int innodb_init_params() sql_print_error("InnoDB: innodb_log_write_ahead_size=%u" " is not a power of two", log_sys.write_size); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } if (compression_algorithm_is_not_loaded(innodb_compression_algorithm, ME_ERROR_LOG)) - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; if ((srv_encrypt_tables || srv_encrypt_log || innodb_encrypt_temporary_tables) && @@ -3853,7 +3874,7 @@ static int innodb_init_params() { sql_print_error("InnoDB: cannot enable encryption, " "encryption plugin is not available"); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } #ifdef _WIN32 @@ -3862,7 +3883,7 @@ static int innodb_init_params() { sql_print_error("InnoDB: innodb_buffer_pool_filename" " cannot have colon (:) in the file name."); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } #endif @@ -3919,7 +3940,7 @@ static int innodb_init_params() { sql_print_error("InnoDB: Unable to parse innodb_data_file_path=%s", innobase_data_file_path); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } srv_tmp_space.set_path(srv_data_home); @@ -3928,7 +3949,7 @@ static int innodb_init_params() { sql_print_error("InnoDB: Unable to parse innodb_temp_data_file_path=%s", innobase_temp_data_file_path); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } /* Perform all sanity check before we take action of deleting files*/ @@ -3936,7 +3957,7 @@ static int innodb_init_params() { sql_print_error("innodb_temporary and innodb_system" " file names seem to be the same."); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } srv_sys_space.normalize_size(); @@ -3949,17 +3970,14 @@ static int innodb_init_params() if (strchr(srv_undo_dir, ';')) { sql_print_error("syntax error in innodb_undo_directory"); - DBUG_RETURN(HA_ERR_INITIALIZATION); + goto err; } if (!srv_log_group_home_dir) srv_log_group_home_dir= const_cast(fil_path_to_mysql_datadir); - if (strchr(srv_log_group_home_dir, ';')) - { - sql_print_error("syntax error in innodb_log_group_home_dir"); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } + srv_log_group_home_dir= my_strdup(PSI_NOT_INSTRUMENTED, + srv_log_group_home_dir, MYF(MY_FAE)); /* Check that interdependent parameters have sane values. */ if (srv_max_buf_pool_modified_pct < srv_max_dirty_pages_pct_lwm) @@ -4331,6 +4349,7 @@ innobase_end(handlerton*, ha_panic_function) innodb_shutdown(); mysql_mutex_destroy(&log_requests.mutex); + my_free(srv_log_group_home_dir); } DBUG_RETURN(0); @@ -18406,44 +18425,6 @@ static uint innodb_merge_threshold_set_all_debug = DICT_INDEX_MERGE_THRESHOLD_DEFAULT; #endif -/** Force an InnoDB log checkpoint. */ -static -void -checkpoint_now_set(THD* thd, st_mysql_sys_var*, void*, const void *save) -{ - if (!*static_cast(save)) - return; - - if (srv_read_only_mode) - { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB doesn't force checkpoint " - "when %s", - (srv_force_recovery - == SRV_FORCE_NO_LOG_REDO) - ? "innodb-force-recovery=6." - : "innodb-read-only=1."); - return; - } - - const auto size= log_sys.is_encrypted() - ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; - mysql_mutex_unlock(&LOCK_global_system_variables); - while (!thd_kill_level(thd)) - { - log_sys.latch.wr_lock(SRW_LOCK_CALL); - lsn_t cp= log_sys.last_checkpoint_lsn.load(std::memory_order_relaxed), - lsn= log_sys.get_lsn(); - log_sys.latch.wr_unlock(); - if (cp + size >= lsn) - break; - log_make_checkpoint(); - } - - mysql_mutex_lock(&LOCK_global_system_variables); -} - #ifdef UNIV_DEBUG /****************************************************************//** Force a dirty pages flush now. */ @@ -18639,74 +18620,167 @@ static void innodb_doublewrite_update(THD *, st_mysql_sys_var*, fil_system.set_use_doublewrite(*static_cast(save)); } -static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, - void *var, const void *save) -{ - ut_ad(var == &srv_log_file_size); - mysql_mutex_unlock(&LOCK_global_system_variables); - - if (high_level_read_only) - ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_READ_ONLY_MODE); +/** Update innodb_log_file_size, innodb_log_file_disabled, +innodb_log_checkpoint_now=ON, innodb_log_group_home_dir. +@param thd execution context +@param spec the specified parameters +@param size innodb_log_file_size (if spec & group::LOG_SIZE) +@param disabled innodb_log_file_disabled (if spec & group::LOG_DISABLED) +@param checkpt innodb_log_checkpoint_now (if spec & group::LOG_CHECKPOINT) +@param dir innodb_log_group_home_dir (if spec & group::LOG_DIR) */ +static void innodb_log_update(THD *thd, unsigned spec, + ulonglong size, bool disabled, bool checkpt, + const char *dir) +{ + switch (spec) { + case group::LOG_CHECKPOINT: + if (checkpt) + { + const auto size= log_sys.is_encrypted() + ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; + mysql_mutex_unlock(&LOCK_global_system_variables); + while (!thd_kill_level(thd)) + { + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t cp= log_sys.last_checkpoint_lsn.load(std::memory_order_relaxed), + lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + if (cp + size >= lsn) + break; + log_make_checkpoint(); + } + mysql_mutex_lock(&LOCK_global_system_variables); + } + /* fall through */ + case group::NONE: + return; + } + log_sys.latch.rd_lock(SRW_LOCK_CALL); + if (!(spec & group::LOG_SIZE)) + size= log_sys.file_size; else if ( #ifdef HAVE_PMEM - !log_sys.is_mmap() && + !log_sys.is_mmap() && #endif - *static_cast(save) < log_sys.buf_size) + size < log_sys.buf_size_requested) + { + log_sys.latch.rd_unlock(); my_printf_error(ER_WRONG_ARGUMENTS, "innodb_log_file_size must be at least" - " innodb_log_buffer_size=%u", MYF(0), log_sys.buf_size); - else + " innodb_log_buffer_size=%u", MYF(0), + log_sys.buf_size_requested); + return; + } + if (!(spec & group::LOG_DISABLED)) + disabled= log_sys.disabled; + if (!(spec & group::LOG_DIR)) + dir= srv_log_group_home_dir; + else if (!dir) + /* SET GLOBAL innodb_log_group_home_dir=DEFAULT */ + dir= const_cast(fil_path_to_mysql_datadir); + + log_sys.latch.rd_unlock(); + + if (disabled) { - switch (log_sys.resize_start(*static_cast(save), thd)) { - case log_t::RESIZE_NO_CHANGE: - break; - case log_t::RESIZE_IN_PROGRESS: + if (log_sys.disable(size, dir)) my_printf_error(ER_WRONG_USAGE, - "innodb_log_file_size change is already in progress", - MYF(0)); - break; - case log_t::RESIZE_FAILED: - ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); - break; - case log_t::RESIZE_STARTED: - for (timespec abstime;;) + "innodb_log_file_* change is in progress", MYF(0)); + return; + } + + mysql_mutex_unlock(&LOCK_global_system_variables); + switch (log_sys.resize_start(size, dir, thd)) { + case log_t::RESIZE_NO_CHANGE: + break; + case log_t::RESIZE_IN_PROGRESS: + my_printf_error(ER_WRONG_USAGE, + "innodb_log_file_* change is already in progress", + MYF(0)); + break; + case log_t::RESIZE_FAILED: + ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); + break; + case log_t::RESIZE_STARTED: + for (timespec abstime;;) + { + if (thd_kill_level(thd)) { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } + log_sys.resize_abort(thd); + break; + } - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(SRW_LOCK_CALL); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= log_sys.resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= log_sys.resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_lock(SRW_LOCK_CALL); + if (!resizing || !log_sys.resize_running(thd)) + { + const bool fail{log_sys.disabled || log_sys.file_size != size || + strcmp(srv_log_group_home_dir, dir)}; log_sys.latch.wr_unlock(); + if (fail) + my_printf_error(ER_WRONG_USAGE, + "innodb_log_file_* change was aborted", MYF(0)); + break; + } + while (resizing > log_sys.get_lsn()) + { +#ifdef HAVE_PMEM + ut_ad(!log_sys.is_mmap()); +#endif + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr; + mtr.start(); + mtr.commit_files(log_sys.last_checkpoint_lsn); } + log_sys.latch.wr_unlock(); } } mysql_mutex_lock(&LOCK_global_system_variables); } +static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, + void *ut_d(var), const void *save) +{ + ut_ad(var == &srv_log_file_size); + innodb_log_update(thd, group::LOG_SIZE, + *static_cast(save), false, false, + nullptr); +} + +static void innodb_log_file_disabled_update(THD *thd, st_mysql_sys_var*, + void *ut_d(var), const void *save) +{ + ut_ad(var == &log_sys.disabled); + innodb_log_update(thd, group::LOG_DISABLED, + 0, *static_cast(save), false, nullptr); +} + +static void innodb_log_checkpoint_now_update(THD *thd, st_mysql_sys_var*, + void *, const void *save) +{ + innodb_log_update(thd, group::LOG_CHECKPOINT, + 0, false, *static_cast(save), nullptr); +} + +static void innodb_log_group_home_dir_update(THD *thd, st_mysql_sys_var*, + void *ut_d(var), const void *save) +{ + ut_ad(var == &srv_log_group_home_dir); + innodb_log_update(thd, group::LOG_DIR, + 0, false, false, *static_cast(save)); +} + /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -19056,7 +19130,7 @@ static MYSQL_SYSVAR_ULONG(io_capacity_max, srv_max_io_capacity, static MYSQL_SYSVAR_BOOL(log_checkpoint_now, innodb_log_checkpoint_now, PLUGIN_VAR_OPCMDARG, "Write back dirty pages from the buffer pool and update the log checkpoint", - NULL, checkpoint_now_set, FALSE); + nullptr, innodb_log_checkpoint_now_update, FALSE); #ifdef UNIV_DEBUG static MYSQL_SYSVAR_BOOL(buf_flush_list_now, innodb_buf_flush_list_now, @@ -19142,8 +19216,9 @@ static MYSQL_SYSVAR_ENUM(flush_method, innodb_flush_method, NULL, NULL, innodb_flush_method_default, &innodb_flush_method_typelib); static MYSQL_SYSVAR_STR(log_group_home_dir, srv_log_group_home_dir, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Path to ib_logfile0", NULL, NULL, NULL); + PLUGIN_VAR_RQCMDARG, + "Path to ib_logfile0", + nullptr, innodb_log_group_home_dir_update, NULL); static MYSQL_SYSVAR_DOUBLE(max_dirty_pages_pct, srv_max_buf_pool_modified_pct, PLUGIN_VAR_RQCMDARG, @@ -19195,7 +19270,8 @@ static MYSQL_SYSVAR_ULONG(max_purge_lag_delay, srv_max_purge_lag_delay, static MYSQL_SYSVAR_UINT(max_purge_lag_wait, innodb_max_purge_lag_wait, PLUGIN_VAR_RQCMDARG, "Wait until History list length is below the specified limit", - NULL, innodb_max_purge_lag_wait_update, UINT_MAX, 0, UINT_MAX, 0); + nullptr, innodb_max_purge_lag_wait_update, + UINT_MAX, 0, UINT_MAX, 0); static MYSQL_SYSVAR_BOOL(rollback_on_timeout, innobase_rollback_on_timeout, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, @@ -19498,11 +19574,16 @@ static MYSQL_SYSVAR_ULONG(page_size, srv_page_size, NULL, NULL, UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE_MIN, UNIV_PAGE_SIZE_MAX, 0); -static MYSQL_SYSVAR_UINT(log_buffer_size, log_sys.buf_size, +static MYSQL_SYSVAR_UINT(log_buffer_size, log_sys.buf_size_requested, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Redo log buffer size in bytes", NULL, NULL, 16U << 20, 2U << 20, log_sys.buf_size_max, 4096); +static MYSQL_SYSVAR_BOOL(log_file_disabled, log_sys.disabled, + PLUGIN_VAR_OPCMDARG, + "Whether the ib_logfile0 is disabled (and InnoDB is crash-unsafe)", + nullptr, innodb_log_file_disabled_update, FALSE); + static constexpr const char *innodb_log_file_mmap_description= "Whether ib_logfile0" " resides in persistent memory (when supported) or" @@ -19983,6 +20064,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(deadlock_report), MYSQL_SYSVAR(page_size), MYSQL_SYSVAR(log_buffer_size), + MYSQL_SYSVAR(log_file_disabled), MYSQL_SYSVAR(log_file_mmap), #if defined __linux__ || defined _WIN32 MYSQL_SYSVAR(log_file_buffering), diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index ce6a4f2779456..3064848a60746 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -39,17 +39,11 @@ static const char LOG_FILE_NAME_PREFIX[] = "ib_logfile"; static const char LOG_FILE_NAME[] = "ib_logfile0"; /** Composes full path for a redo log file -@param[in] filename name of the redo log file +@param filename name of the redo log file +@param directory name of the directory; nullptr=innodb_log_group_home_dir @return path with log file name*/ -std::string get_log_file_path(const char *filename= LOG_FILE_NAME); - -/** Delete log file. -@param[in] suffix suffix of the file name */ -static inline void delete_log_file(const char* suffix) -{ - auto path = get_log_file_path(LOG_FILE_NAME_PREFIX).append(suffix); - os_file_delete_if_exists_func(path.c_str(), nullptr); -} +std::string get_log_file_path(const char *filename= LOG_FILE_NAME, + const char *directory= nullptr); struct completion_callback; @@ -130,7 +124,7 @@ class log_file_t /** Redo log buffer */ struct log_t { - /** The maximum buf_size */ + /** The maximum buf_size_requested */ static constexpr unsigned buf_size_max= os_file_request_size_max; /** The original (not version-tagged) InnoDB redo log format */ @@ -182,7 +176,7 @@ struct log_t /** the first guaranteed-durable log sequence number */ std::atomic flushed_to_disk_lsn; public: - /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ + /** innodb_log_buffer_size or the PMEM file_size */ unsigned buf_size; /** log file size in bytes, including the header */ lsn_t file_size; @@ -255,7 +249,11 @@ struct log_t lsn_t resize_target; /** Buffer for writing to resize_log; @see buf */ byte *resize_buf; - /** Buffer for writing to resize_log; @see flush_buf */ + /** Buffer for writing to resize_log; @see flush_buf. + When a PMEM based ib_logfile101 is being created during + SET GLOBAL innodb_log_file_disabled=OFF, this will store + the original value of a dummy log_sys.flush_buf that was being used by + skip_write_buf(). */ byte *resize_flush_buf; /** log sequence number when log resizing was initiated; @@ -264,10 +262,14 @@ struct log_t /** the log sequence number at the start of the log file */ lsn_t first_lsn; public: + /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ + unsigned buf_size_requested; /** current innodb_log_write_ahead_size */ uint write_size; /** format of the redo log: e.g., FORMAT_10_8 */ uint32_t format; + /** whether the log is disabled */ + my_bool disabled; /** whether the memory-mapped interface is enabled for the log */ my_bool log_mmap; /** the default value of log_mmap */ @@ -314,8 +316,10 @@ struct log_t /* @} */ private: - /** the thread that initiated resize_lsn() */ - Atomic_relaxed resize_initiator; + /** the desired value of innodb_log_group_home_dir */ + const char *resize_dir; + /** the thread that initiated resize_lsn(); protected by latch */ + void *resize_initiator; #ifdef HAVE_PMEM /** mutex protecting wrap-around in resize_write() */ srw_mutex resize_wrap_mutex; @@ -334,6 +338,13 @@ struct log_t is_mmap() && !is_opened() holds for PMEM */ bool is_opened() const noexcept { return log.is_opened(); } + /** Try to disable the redo log. + @param file_size the desired log_sys.file_size + @param dir the desired innodb_log_group_home_dir + @retval false if the log is or was disabled + @retval true if resize_in_progress() */ + bool disable(lsn_t file_size, const char *dir) noexcept; + /** @return LSN at which log resizing was started and is still in progress @retval 0 if no log resizing is in progress @retval 1 if resize_start() is in progress */ @@ -347,9 +358,11 @@ struct log_t /** Start resizing the log and release the exclusive latch. @param size requested new file_size + @param dir the desired innodb_log_group_home_dir @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(os_offset_t size, const char *dir, + void *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -357,7 +370,7 @@ struct log_t /** @return whether a particular resize_start() is in progress */ bool resize_running(void *thd) const noexcept - { return thd == resize_initiator; } + { ut_ad(latch_have_any()); return thd == resize_initiator; } /** Replicate a write to the log. @param lsn start LSN @@ -434,6 +447,7 @@ struct log_t lsn_t get_flushed_lsn(std::memory_order order= std::memory_order_acquire) const noexcept { return flushed_to_disk_lsn.load(order); } + inline void set_flushed_lsn(lsn_t lsn) noexcept; /** Initialize the LSN on initial log file creation. */ inline lsn_t init_lsn() noexcept; @@ -548,6 +562,10 @@ struct log_t RESIZING }; + /** Pretend to write to (non-existing) ib_logfile0 while disabled==true + @return the current log sequence number */ + static lsn_t skip_write_buf() noexcept; + /** Write buf to ib_logfile0 and possibly ib_logfile101. @tparam resizing whether to release latch and whether resize_in_progress()>1 @return the current log sequence number */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 1bf47bfe14eea..2420cf10f261c 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -324,14 +324,16 @@ static inline void log_file_message() noexcept {} bool log_t::attach(log_file_t file, os_offset_t size) noexcept { - log= file; + ut_ad(disabled == (file.m_file == OS_FILE_CLOSED)); ut_ad(!size || size >= START_OFFSET + SIZE_OF_FILE_CHECKPOINT); - file_size= size; - ut_ad(!buf); ut_ad(!flush_buf); ut_ad(!writer); - if (size) + + log= file; + file_size= size; + + if (size && !disabled) { # ifdef HAVE_PMEM bool is_pmem; @@ -360,6 +362,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) noexcept } } log_mmap= false; + buf_size= buf_size_requested; buf= static_cast(ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME)); if (!buf) { @@ -393,6 +396,8 @@ bool log_t::attach(log_file_t file, os_offset_t size) noexcept TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); writer_update(false); + if (disabled) + writer= skip_write_buf; memset_aligned<512>(checkpoint_buf, 0, write_size); func_exit: @@ -446,6 +451,7 @@ void log_t::create(lsn_t lsn) noexcept #ifdef HAVE_PMEM if (is_mmap()) { + ut_ad(!disabled); ut_ad(!is_opened()); mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); memset_aligned<4096>(buf, 0, 4096); @@ -455,6 +461,7 @@ void log_t::create(lsn_t lsn) noexcept } else #endif + if (!disabled) { ut_ad(!is_mmap()); memset_aligned<4096>(flush_buf, 0, buf_size); @@ -474,7 +481,6 @@ void log_t::close_file(bool really_close) noexcept { if (is_mmap()) { - ut_ad(!checkpoint_buf); ut_ad(!flush_buf); if (buf) { @@ -485,7 +491,6 @@ void log_t::close_file(bool really_close) noexcept else { ut_ad(!buf == !flush_buf); - ut_ad(!buf == !checkpoint_buf); if (buf) { ut_free_dodump(buf, buf_size); @@ -493,6 +498,12 @@ void log_t::close_file(bool really_close) noexcept ut_free_dodump(flush_buf, buf_size); flush_buf= nullptr; } + } + +#ifdef HAVE_PMEM + if (checkpoint_buf) +#endif + { aligned_free(checkpoint_buf); checkpoint_buf= nullptr; } @@ -514,20 +525,38 @@ lsn_t log_get_lsn() noexcept return lsn; } +/** Acquire the group locks. */ +static void log_resize_acquire_group() noexcept +{ + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != + group_commit_lock::ACQUIRED); + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != + group_commit_lock::ACQUIRED); +} + /** Acquire all latches that protect the log. */ -static void log_resize_acquire() noexcept +static bool log_resize_acquire() noexcept { + bool skip_group_lock= #ifdef HAVE_PMEM - if (!log_sys.is_mmap()) + log_sys.is_mmap() || #endif - { - while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != - group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_get_lsn() + 1, nullptr) != - group_commit_lock::ACQUIRED); - } + false; + + if (!skip_group_lock) + log_resize_acquire_group(); log_sys.latch.wr_lock(SRW_LOCK_CALL); + return skip_group_lock; +} + +/** Release the group locks. */ +static void log_resize_release_group() noexcept +{ + lsn_t lsn1= write_lock.release(write_lock.value()); + lsn_t lsn2= flush_lock.release(flush_lock.value()); + if (lsn1 || lsn2) + log_write_up_to(std::max(lsn1, lsn2), true, nullptr); } /** Release the latches that protect the log. */ @@ -538,12 +567,7 @@ void log_resize_release() noexcept #ifdef HAVE_PMEM if (!log_sys.is_mmap()) #endif - { - lsn_t lsn1= write_lock.release(write_lock.value()); - lsn_t lsn2= flush_lock.release(flush_lock.value()); - if (lsn1 || lsn2) - log_write_up_to(std::max(lsn1, lsn2), true, nullptr); - } + log_resize_release_group(); } #if defined __linux__ || defined _WIN32 @@ -599,12 +623,160 @@ void log_t::set_write_through(bool write_through) log_resize_release(); } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier -@return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +ATTRIBUTE_COLD static lsn_t log_writer_resizing() noexcept; + +ATTRIBUTE_COLD lsn_t log_t::skip_write_buf() noexcept +{ + ut_ad(log_sys.latch_have_wr()); + ut_ad(log_sys.disabled); + ut_ad(!srv_read_only_mode); + const lsn_t lsn{log_sys.get_lsn()}; + ut_ad(log_sys.write_lsn >= log_sys.get_flushed_lsn()); + const size_t write_size_1{log_sys.write_size - 1}; + ut_ad(ut_is_2pow(log_sys.write_size)); + lsn_t base= log_sys.base_lsn.load(std::memory_order_relaxed); + size_t length{size_t(lsn - base)}; + ut_ad(length >= (log_sys.calc_lsn_offset(log_sys.write_lsn) & write_size_1)); + ut_ad(write_size_1 >= 511); + + if (length <= write_size_1) + ut_ad(!((length ^ (size_t(lsn) - size_t(log_sys.first_lsn))) + & write_size_1)); + else + { + const size_t new_buf_free{length & write_size_1}; + base+= length & ~write_size_1; + ut_ad(new_buf_free == ((lsn - log_sys.first_lsn) & write_size_1)); + log_sys.write_to_buf+= + size_t(log_sys.write_lsn_offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + log_sys.write_lsn_offset.store(new_buf_free, std::memory_order_relaxed); + log_sys.base_lsn.store(base, std::memory_order_release); + + if (new_buf_free) + { + /* Preserve the last partial block block. It may be overwritten + later in write_buf(), once records beyond the current LSN are + generated. */ + MEM_MAKE_DEFINED(log_sys.buf + length, + (write_size_1 + 1) - new_buf_free); + length&= ~write_size_1; + memcpy_aligned<16>(log_sys.flush_buf, log_sys.buf + length, + (new_buf_free + 15) & ~15); + } + + std::swap(log_sys.buf, log_sys.flush_buf); + } + + ut_ad(base + (log_sys.write_lsn_offset & (WRITE_TO_BUF - 1)) == lsn); + log_sys.write_lsn= lsn; + log_sys.set_check_for_checkpoint(false); + log_sys.latch.wr_unlock(); + return lsn; +} + +/** Invoke commit_checkpoint_notify_ha() to notify that outstanding +log writes have been completed. */ +void log_flush_notify(lsn_t flush_lsn); + +bool log_t::disable(lsn_t file_size, const char *dir) noexcept +{ + ut_ad(!srv_read_only_mode); + +#ifdef HAVE_PMEM + log_resize_acquire_group(); + log_sys.latch.wr_lock(SRW_LOCK_CALL); +#else + log_resize_acquire(); +#endif + + if (resize_in_progress()) + { +#ifdef HAVE_PMEM + log_sys.latch.wr_unlock(); + log_resize_release_group(); +#else + log_resize_release(); +#endif + return true; + } + + ut_ad(!resize_log.is_opened()); + ut_ad(!resize_buf); + ut_ad(!resize_flush_buf); + ut_ad(!resize_initiator); + ut_ad(!resize_target); + + if (!disabled) + { + disabled= true; +#ifdef HAVE_PMEM + if (is_mmap()) + { + byte *b= static_cast(ut_malloc_dontdump + (buf_size_requested, PSI_INSTRUMENT_ME)); + if (!b) + { + alloc_fail: + log_sys.latch.wr_unlock(); + log_resize_release_group(); + return true; + } + + flush_buf= static_cast(ut_malloc_dontdump + (buf_size_requested, PSI_INSTRUMENT_ME)); + if (!flush_buf) + { + ut_free_dodump(b, buf_size); + goto alloc_fail; + } + + my_munmap(buf, this->file_size); + buf= b; + buf_size= buf_size_requested; + } + else +#endif + { + ut_ad(!is_mmap()); + log.close(); + } + + writer= skip_write_buf; + mtr_t::finisher_update(); + std::string path{get_log_file_path()}; + IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); + } + ut_ad(writer == skip_write_buf); + ut_ad(!log.is_opened()); + srv_log_file_size= this->file_size= file_size; + if (srv_log_group_home_dir != dir) + { + my_free(srv_log_group_home_dir); + srv_log_group_home_dir= my_strdup(PSI_NOT_INSTRUMENTED, dir, MYF(MY_FAE)); + } + + uint64_t offset{write_lsn_offset}; + lsn_t lsn= write_lsn= first_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(lsn, std::memory_order_release); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + ut_a(!write_lock.release(lsn)); + ut_a(!flush_lock.release(lsn)); + log_sys.latch.wr_unlock(); + log_flush_notify(lsn); + + sql_print_information("InnoDB: Disabled redo log at LSN=" LSN_PF, lsn); + return false; +} + +log_t::resize_start_status +log_t::resize_start(os_offset_t size, const char *dir, void *thd) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -615,19 +787,19 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_start_status status; - if (size == file_size) + if (size == file_size && !disabled && !strcmp(dir, srv_log_group_home_dir)) status= RESIZE_NO_CHANGE; else if (resize_in_progress()) status= RESIZE_IN_PROGRESS; else { lsn_t start_lsn; - ut_ad(!resize_in_progress()); + ut_ad(disabled == (writer == skip_write_buf)); ut_ad(!resize_log.is_opened()); ut_ad(!resize_buf); ut_ad(!resize_flush_buf); ut_ad(!resize_initiator); - std::string path{get_log_file_path("ib_logfile101")}; + std::string path{get_log_file_path("ib_logfile101", dir)}; bool success; resize_initiator= thd; resize_lsn.store(1, std::memory_order_relaxed); @@ -635,6 +807,10 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_log.m_file= os_file_create_func(path.c_str(), OS_FILE_CREATE, OS_LOG_FILE, false, &success); +#ifdef HAVE_PMEM + bool is_pmem{false}; +#endif + if (success) { ut_ad(!(size_t(file_size) & (write_size - 1))); @@ -643,39 +819,61 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) void *ptr= nullptr, *ptr2= nullptr; success= os_file_set_size(path.c_str(), resize_log.m_file, size); - if (!success); -#ifdef HAVE_PMEM - else if (is_mmap()) + if (success) { - bool is_pmem{false}; +#ifdef HAVE_PMEM ptr= ::log_mmap(resize_log.m_file, is_pmem, size); - if (ptr == MAP_FAILED) - goto alloc_fail; - } -#endif - else - { - ut_ad(!is_mmap()); - ptr= ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME); - if (ptr) + if (ptr != MAP_FAILED) { - TRASH_ALLOC(ptr, buf_size); - ptr2= ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME); - if (ptr2) - TRASH_ALLOC(ptr2, buf_size); - else + if (!is_pmem) { - ut_free_dodump(ptr, buf_size); - ptr= nullptr; - goto alloc_fail; + my_munmap(ptr, file_size); + goto no_mmap; } + if (!disabled && !is_mmap()) + goto alloc_fail; } else - alloc_fail: - success= false; + { + no_mmap: + if (is_mmap()) + goto alloc_fail; + if (!checkpoint_buf) + { + checkpoint_buf= + static_cast(aligned_malloc(write_size, write_size)); + if (!checkpoint_buf) + goto alloc_fail; + memset_aligned<512>(checkpoint_buf, 0, write_size); + } +#endif + ut_ad(!is_mmap()); + ptr= ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME); + if (ptr) + { + TRASH_ALLOC(ptr, buf_size); + ptr2= ut_malloc_dontdump(buf_size, PSI_INSTRUMENT_ME); + if (ptr2) + TRASH_ALLOC(ptr2, buf_size); + else + { + ut_free_dodump(ptr, buf_size); + ptr= nullptr; + goto alloc_fail; + } + } + else + alloc_fail: + success= false; +#ifdef HAVE_PMEM + } +#endif } +#ifdef HAVE_PMEM + bool skip_group_lock= +#endif log_resize_acquire(); if (!success) @@ -688,18 +886,58 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; resize_buf= static_cast(ptr); resize_flush_buf= static_cast(ptr2); - start_lsn= get_lsn(); - if (!is_mmap()) + uint64_t offset{write_lsn_offset}; + start_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + +#ifdef HAVE_PMEM + if (is_pmem) + { + resize_log.close(); + if (disabled) + { + /* Clear flush_buf to pretend to use memory-mapped log writes + to both buffers. For resize_abort(), store the old value + in resize_flush_buf. */ + resize_flush_buf= flush_buf; + flush_buf= nullptr; + goto resize_and_enable; + } + } + else +#endif + if (disabled) + { +#ifdef HAVE_PMEM + resize_and_enable: +#endif + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); + /* The following synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(start_lsn, std::memory_order_release); + + flushed_to_disk_lsn.store(start_lsn, std::memory_order_relaxed); + + write_lsn= start_lsn; + first_lsn= start_lsn; + } + else start_lsn= first_lsn + (~lsn_t{write_size - 1} & (lsn_t{write_size - 1} + start_lsn - first_lsn)); - else if (!is_opened()) - resize_log.close(); resize_lsn.store(start_lsn, std::memory_order_relaxed); + resize_dir= dir; writer_update(true); +#ifdef HAVE_PMEM + latch.wr_unlock(); + if (!skip_group_lock) + log_resize_release_group(); +#else log_resize_release(); +#endif mysql_mutex_lock(&buf_pool.flush_list_mutex); lsn_t target_lsn= buf_pool.get_oldest_modification(0); @@ -721,42 +959,92 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { +#ifdef HAVE_PMEM + log_resize_acquire_group(); + log_sys.latch.wr_lock(SRW_LOCK_CALL); +#else log_resize_acquire(); +#endif - if (resize_running(thd)) + if (!resize_running(thd)) { + aborted: #ifdef HAVE_PMEM - const bool is_mmap{this->is_mmap()}; + log_sys.latch.wr_unlock(); + log_resize_release_group(); #else - constexpr bool is_mmap{false}; + log_resize_release(); #endif - if (!is_mmap) + return; + } + + resize_lsn.store(0, std::memory_order_relaxed); + resize_initiator= nullptr; + lsn_t lsn= 0; +#ifdef HAVE_PMEM + if (!flush_buf) + { + ut_ad(!!resize_flush_buf == disabled); + ut_ad(!resize_log.is_opened()); + ut_ad(resize_buf); + + /* Restore the dummy flush_buf that resize_start() had stored here. */ + flush_buf= resize_flush_buf; + resize_flush_buf= nullptr; + + my_munmap(resize_buf, resize_target); + + if (disabled) { - ut_free_dodump(resize_buf, buf_size); - ut_free_dodump(resize_flush_buf, buf_size); - resize_flush_buf= nullptr; + writer= skip_write_buf; + mtr_t::finisher_update(); + goto disabled; } + } + else +#endif + { + ut_ad(flush_buf != nullptr); + ut_ad(resize_buf != nullptr); + ut_ad(resize_flush_buf != nullptr); + ut_free_dodump(resize_buf, buf_size); + ut_free_dodump(resize_flush_buf, buf_size); + resize_flush_buf= nullptr; + if (UNIV_LIKELY(!disabled)) + writer_update(false); else { - ut_ad(!resize_log.is_opened()); - ut_ad(!resize_flush_buf); + ut_ad(writer == skip_write_buf || writer == log_writer_resizing); + writer= skip_write_buf; #ifdef HAVE_PMEM - if (resize_buf) - my_munmap(resize_buf, resize_target); -#endif /* HAVE_PMEM */ + disabled: +#endif + uint64_t offset{write_lsn_offset}; + lsn= write_lsn= first_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); } - if (resize_log.is_opened()) - resize_log.close(); - resize_buf= nullptr; - resize_target= 0; - resize_lsn.store(0, std::memory_order_relaxed); - resize_initiator= nullptr; - std::string path{get_log_file_path("ib_logfile101")}; - IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); - writer_update(false); } + if (resize_log.is_opened()) + resize_log.close(); + resize_buf= nullptr; + resize_target= 0; + std::string path{get_log_file_path("ib_logfile101", resize_dir)}; + resize_dir= nullptr; + IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); - log_resize_release(); + if (!lsn) + goto aborted; + + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(lsn, std::memory_order_release); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + ut_a(!write_lock.release(lsn)); + ut_a(!flush_lock.release(lsn)); + log_sys.latch.wr_unlock(); + log_flush_notify(lsn); } /** Write an aligned buffer to ib_logfile0. @@ -787,10 +1075,6 @@ static void log_write_buf(const byte *buf, size_t length, lsn_t offset) log_sys.log.write(offset, {buf, length}); } -/** Invoke commit_checkpoint_notify_ha() to notify that outstanding -log writes have been completed. */ -void log_flush_notify(lsn_t flush_lsn); - #if 0 // Currently we overwrite the last log block until it is complete. /** CRC-32C of pad messages using between 1 and 15 bytes of NUL bytes in the payload */ @@ -916,8 +1200,6 @@ static size_t log_pad(lsn_t lsn, size_t pad, byte *begin, byte *extra) void log_t::persist(lsn_t lsn) noexcept { ut_ad(!is_opened()); - ut_ad(!write_lock.is_owner()); - ut_ad(!flush_lock.is_owner()); ut_ad(latch_have_wr()); lsn_t old= flushed_to_disk_lsn.load(std::memory_order_relaxed); @@ -925,16 +1207,19 @@ void log_t::persist(lsn_t lsn) noexcept if (old >= lsn) return; - const size_t start(calc_lsn_offset(old)); - const size_t end(calc_lsn_offset(lsn)); - - if (UNIV_UNLIKELY(end < start)) + if (UNIV_LIKELY(disabled)) { - pmem_persist(buf + start, file_size - start); - pmem_persist(buf + START_OFFSET, end - START_OFFSET); + const size_t start(calc_lsn_offset(old)); + const size_t end(calc_lsn_offset(lsn)); + + if (UNIV_UNLIKELY(end < start)) + { + pmem_persist(buf + start, file_size - start); + pmem_persist(buf + START_OFFSET, end - START_OFFSET); + } + else + pmem_persist(buf + start, end - start); } - else - pmem_persist(buf + start, end - start); uint64_t offset{write_lsn_offset}; const lsn_t new_base_lsn= base_lsn.load(std::memory_order_relaxed) + @@ -1001,10 +1286,15 @@ inline __attribute__((always_inline)) lsn_t log_t::write_buf() noexcept { ut_ad(latch_have_wr()); +#ifdef HAVE_PMEM + ut_ad(resizing == RESIZING || !is_mmap()); +#else ut_ad(!is_mmap()); +#endif ut_ad(!srv_read_only_mode); ut_ad(resizing == RETAIN_LATCH || (resizing == RESIZING) == (resize_in_progress() > 1)); + ut_ad(resizing != NOT_RESIZING || !disabled); const lsn_t lsn{get_lsn()}; @@ -1014,6 +1304,17 @@ lsn_t log_t::write_buf() noexcept latch.wr_unlock(); ut_ad(write_lsn == lsn); } +#ifdef HAVE_PMEM + else if (resizing == RESIZING && is_mmap()) + { + ut_ad(write_lock.is_owner()); + ut_ad(!recv_no_log_write); + ut_ad(!resize_log.is_opened()); + ut_ad(disabled); + latch.wr_unlock(); + write_lsn= lsn; + } +#endif else { ut_ad(write_lock.is_owner()); @@ -1102,7 +1403,8 @@ lsn_t log_t::write_buf() noexcept write_lsn, lsn, offset)); /* Do the write to the log file */ - log_write_buf(write_buf, length, offset); + if (resizing == NOT_RESIZING || !disabled) + log_write_buf(write_buf, length, offset); if (UNIV_LIKELY_NULL(re_write_buf)) resize_write_buf(re_write_buf, length); @@ -1123,7 +1425,7 @@ bool log_t::flush(lsn_t lsn) noexcept { ut_ad(lsn >= get_flushed_lsn()); flush_lock.set_pending(lsn); - const bool success{log_write_through || log.flush()}; + const bool success{log_write_through || disabled || log.flush()}; if (UNIV_LIKELY(success)) { flushed_to_disk_lsn.store(lsn, std::memory_order_release); @@ -1148,6 +1450,13 @@ static lsn_t log_flush(lsn_t lsn) noexcept static const completion_callback dummy_callback{[](void *) {},nullptr}; +inline void log_t::set_flushed_lsn(lsn_t lsn) noexcept +{ + ut_ad(flushed_to_disk_lsn.load(std::memory_order_relaxed) <= lsn); + flushed_to_disk_lsn.store(lsn, std::memory_order_release); + log_flush_notify(lsn); +} + /** Ensure that the log has been written to the log file up to a given log entry (such as that of a transaction commit). Start a new write, or wait and check if an already running write is covering the request. @@ -1194,10 +1503,26 @@ void log_write_up_to(lsn_t lsn, bool durable, pending_write_lsn= write_lock.release(log_sys.writer()); } - if (durable) + if (!durable); +#ifdef HAVE_PMEM + else if (UNIV_UNLIKELY(log_sys.is_mmap())) { - pending_flush_lsn= log_flush(write_lock.value()); + ut_ad(log_sys.resize_in_progress()); + goto skip_write; } +#endif + else if (UNIV_UNLIKELY(log_sys.disabled)) + { + ut_ad(flush_lock.is_owner()); +#ifdef HAVE_PMEM + skip_write: +#endif + lsn= std::max(write_lock.value(), log_sys.get_flushed_lsn()); + log_sys.set_flushed_lsn(lsn); + pending_flush_lsn= flush_lock.release(lsn); + } + else + pending_flush_lsn= log_flush(write_lock.value()); if (pending_write_lsn || pending_flush_lsn) { @@ -1236,11 +1561,6 @@ void log_buffer_flush_to_disk(bool durable) noexcept /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept { -#ifdef HAVE_PMEM - if (log_sys.is_mmap()) - return; -#endif - while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); while (write_lock.acquire(log_get_lsn() + 1, nullptr) != @@ -1294,7 +1614,12 @@ ATTRIBUTE_COLD void log_write_and_flush() noexcept ut_ad(!srv_read_only_mode); #ifdef HAVE_PMEM if (log_sys.is_mmap()) - log_sys.persist(log_sys.get_lsn()); + { + const lsn_t lsn{log_sys.get_lsn()}; + log_sys.persist(lsn); + write_lock.release(lsn); + flush_lock.release(lsn); + } else #endif { @@ -1583,13 +1908,15 @@ void log_t::close() recv_sys.close(); } -std::string get_log_file_path(const char *filename) +std::string get_log_file_path(const char *filename, const char *dir) { - const size_t size= strlen(srv_log_group_home_dir) + /* path separator */ 1 + + if (!dir) + dir= srv_log_group_home_dir; + const size_t size= strlen(dir) + /* path separator */ 1 + strlen(filename) + /* longest suffix */ 3; std::string path; path.reserve(size); - path.assign(srv_log_group_home_dir); + path.assign(dir); switch (path.back()) { #ifdef _WIN32 diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 6d67cfa6b0734..491d897cf8355 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -957,7 +957,8 @@ std::pair log_t::append_prepare(size_t size, bool ex) noexcept { ut_ad(ex ? latch_have_wr() : latch_have_rd()); ut_ad(mmap == is_mmap()); - ut_ad(!mmap || buf_size == std::min(capacity(), buf_size_max)); + ut_ad(!mmap || disabled || + buf_size == std::min(capacity(), buf_size_max)); const size_t buf_size{this->buf_size - size}; uint64_t l; static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); @@ -971,6 +972,12 @@ std::pair log_t::append_prepare(size_t size, bool ex) noexcept /* Subtract our LSN overshoot. */ write_lsn_offset.fetch_sub(size); append_prepare_wait(late, ex); +#ifdef HAVE_PMEM + /* While we were waiting, a change of log_sys.disabled could have been + initiated or completed. */ + if (UNIV_UNLIKELY(mmap != is_mmap())) + return append_prepare(size, ex); +#endif } const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}, @@ -979,8 +986,8 @@ std::pair log_t::append_prepare(size_t size, bool ex) noexcept if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) set_check_for_checkpoint(true); - return {lsn, - buf + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; + return {lsn, buf + + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % this->buf_size : l)}; } /** Finish appending data to the log. @@ -1122,8 +1129,9 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, size_t s; #ifdef HAVE_PMEM - if (!resize_flush_buf) + if (!resize_log.is_opened()) { + ut_ad(!resize_flush_buf || disabled); ut_ad(is_mmap()); resize_wrap_mutex.wr_lock(); const size_t resize_capacity{resize_target - START_OFFSET}; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 4d2f72196bc0c..64b3bd1ca0f27 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -160,7 +160,8 @@ static PSI_stage_info* srv_stages[] = static void delete_log_files() { for (size_t i= 1; i < 102; i++) - delete_log_file(std::to_string(i).c_str()); + os_file_delete_if_exists_func(get_log_file_path(LOG_FILE_NAME_PREFIX). + append(std::to_string(i)).c_str(), nullptr); } /** Creates log file. @@ -168,7 +169,7 @@ static void delete_log_files() @param lsn log sequence number @param logfile0 name of the log file @return DB_SUCCESS or error code */ -static dberr_t create_log_file(bool create_new_db, lsn_t lsn) +static dberr_t create_log_file(bool create_new_db, lsn_t lsn) noexcept { ut_ad(!srv_read_only_mode); @@ -194,11 +195,15 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) std::string logfile0{get_log_file_path("ib_logfile101")}; bool ret; - os_file_t file{ - os_file_create_func(logfile0.c_str(), - OS_FILE_CREATE, - OS_LOG_FILE, false, &ret) - }; + os_file_t file; + + if (UNIV_UNLIKELY(log_sys.disabled)) { + file = OS_FILE_CLOSED; + goto file_created; + } + + file = os_file_create_func(logfile0.c_str(), OS_FILE_CREATE, + OS_LOG_FILE, false, &ret); if (!ret) { sql_print_error("InnoDB: Cannot create %.*s", @@ -217,6 +222,7 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) goto err_exit; } +file_created: log_sys.set_latest_format(srv_encrypt_log); if (!log_sys.attach(file, srv_log_file_size)) { goto close_and_exit; @@ -265,18 +271,61 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn) @return whether an error occurred */ bool log_t::resize_rename() noexcept { - std::string old_name{get_log_file_path("ib_logfile101")}; + ut_ad(!log_sys.resize_dir || log_sys.latch_have_wr()); + std::string old_name{get_log_file_path("ib_logfile101", log_sys.resize_dir)}; std::string new_name{get_log_file_path()}; if (IF_WIN(MoveFileEx(old_name.c_str(), new_name.c_str(), MOVEFILE_REPLACE_EXISTING), !rename(old_name.c_str(), new_name.c_str()))) + { + if (log_sys.resize_dir) + { + old_name= get_log_file_path("ib_logfile0", log_sys.resize_dir); + if (IF_WIN(MoveFileEx(new_name.c_str(), old_name.c_str(), + MOVEFILE_REPLACE_EXISTING), + !rename(new_name.c_str(), old_name.c_str()))); + else + { + /* We succeeded in atomically renaming the file to the + original innodb_log_group_home_dir, but not back to the new + innodb_log_group_home_dir. + Retain the current innodb_log_group_home_dir. */ + if (srv_log_group_home_dir != log_sys.resize_dir) + { + my_free(const_cast(log_sys.resize_dir)); + log_sys.resize_dir= srv_log_group_home_dir; + } + } + } return false; + } + const int err= IF_WIN(int(GetLastError()), errno); + const char *remove= old_name.c_str(); + + if (log_sys.resize_dir && err == IF_WIN(ERROR_NOT_SAME_DEVICE, EXDEV)) + { + /* The innodb_log_group_home_dir will point to a different file system. + Try to rename the new file to ib_logfile0 in thew new location. */ + std::string name{get_log_file_path("ib_logfile0", log_sys.resize_dir)}; + if (IF_WIN(MoveFileEx(remove, name.c_str(), MOVEFILE_REPLACE_EXISTING), + !rename(remove, name.c_str()))) + { + /* Now we have two ib_logfile0, both of them valid for recovery. + Remove the one at the old innodb_log_group_home_dir location. */ + IF_WIN(DeleteFile(new_name.c_str()), unlink(new_name.c_str())); + return false; + } + } + else if (log_sys.disabled) + remove= new_name.c_str(); + IF_WIN(DeleteFile(remove), unlink(remove)); + if (log_sys.disabled) + return false; sql_print_error("InnoDB: Failed to rename log from %.*s to %.*s (error %d)", int(old_name.size()), old_name.data(), - int(new_name.size()), new_name.data(), - IF_WIN(int(GetLastError()), errno)); + int(new_name.size()), new_name.data(), err); return true; }