diff --git a/mysql-test/suite/innodb/include/check_core_dump_trim.inc b/mysql-test/suite/innodb/include/check_core_dump_trim.inc new file mode 100644 index 0000000000000..7f3ebec0f5b65 --- /dev/null +++ b/mysql-test/suite/innodb/include/check_core_dump_trim.inc @@ -0,0 +1,104 @@ +# ------------------------------------------------------------------------------ +# Test Summary +# ------------------------------------------------------------------------------ +# This test checks whether the mariadbd core dump file is expected to be TRIMMED +# or NOT TRIMMED based on how memory is mapped by the mariadbd process. +# +# What this script does: +# +# 1. It retrieves MariaDB runtime values: +# - pid_file (location of mariadbd PID) +# - innodb_buffer_pool_size +# +# 2. Using the PID file, it finds the running mariadbd process ID. +# +# 3. It reads /proc//smaps to inspect the memory regions allocated +# by the mariadbd process. +# +# 4. From the smaps output, it identifies candidate memory regions that: +# - are smaller than 192M +# - contain the "dd" VmFlag (indicating the region is eligible for +# core dump trimming) +# +# 5. It sums the size of these candidate memory regions. +# +# 6. If the candidate memory is large enough to cover the buffer pool, +# the script concludes that the buffer pool will not be part of core dump. +# Otherwise, it reports it to be part of core dump. +# +# Purpose: +# This helps verify whether InnoDB buffer pool will be excluded from the +# core dump, preventing unnecessarily large core files. +# ------------------------------------------------------------------------------ + +# Get the full path name of the PID file +--let $pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let PIDFILE= $pid_file + +# Get innodb_buffer_pool_size +--let $innodb_buffer_pool_size= query_get_value(SELECT @@innodb_buffer_pool_size, @@innodb_buffer_pool_size, 1) +--let BUFFER_POOL_SIZE= $innodb_buffer_pool_size + +perl; + +use strict; +use warnings; + +my $pid_file = $ENV{'PIDFILE'} or die "PIDFILE not set"; + +# Buffer pool size +my $buffer_pool_size = $ENV{'BUFFER_POOL_SIZE'}; + +# Convert to KB +my $buffer_pool_kb = int($buffer_pool_size / 1024); + +# Ignore regions larger than 192M +my $max_region_kb = 192 * 1024; + +# Sum of candidate regions +my $candidate_sum_kb = 0; + +# Get PID of mariadbd +open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n"; +my $pid = <$fh>; +$pid =~ s/\s//g; +close($fh); + +if ($pid eq "") { + die "Couldn't retrieve PID from PID file.\n"; +} + +# Open smaps +my $smaps_file = "/proc/$pid/smaps"; + +open(my $sm, '<', $smaps_file) or die "Cannot open $smaps_file\n"; + +my $current_size_kb = 0; + +while (my $line = <$sm>) { + + if ($line =~ /^Size:\s+(\d+)\s+kB/) { + $current_size_kb = $1; + } + + if ($line =~ /^VmFlags:\s+(.*)/) { + my $flags = $1; + + # Candidate region conditions + if ($current_size_kb <= $max_region_kb && + $flags =~ /\bdd\b/) { + $candidate_sum_kb += $current_size_kb; + } + } +} + +close($sm); + +if ($candidate_sum_kb >= $buffer_pool_kb) { + print "Buffer pool not part of core dump.\n"; +} +else { + print "Buffer pool part of core dump.\n"; +} + +EOF diff --git a/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result b/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result new file mode 100644 index 0000000000000..ac64260b85cad --- /dev/null +++ b/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result @@ -0,0 +1,31 @@ +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +# +# Confirm innodb_buffer_pool_in_core_dump works +# +SELECT @@global.innodb_buffer_pool_in_core_dump; +@@global.innodb_buffer_pool_in_core_dump +0 +Buffer pool not part of core dump. +# +# Confirm dynamicity of innodb_buffer_pool_in_core_dump +# +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +Buffer pool part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +Buffer pool not part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +Buffer pool part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +Buffer pool not part of core dump. +# +# Confirm innodb_pool_in_core_dump works even +# after resizing of buffer pool. +# +set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size; +set global innodb_buffer_pool_size = 64*1024*1024; +Buffer pool not part of core dump. +set global innodb_buffer_pool_size = 96*1024*1024; +Buffer pool not part of core dump. +set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size; +set global innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; +# End of 10.11 tests diff --git a/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt new file mode 100644 index 0000000000000..c388fc9c426b4 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt @@ -0,0 +1,2 @@ +--loose-skip-innodb-buffer-pool-in-core-dump +--innodb-buffer-pool-size=128M diff --git a/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test new file mode 100644 index 0000000000000..e22c1ed7aed19 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test @@ -0,0 +1,56 @@ +# +# MDEV-22186 : Add innodb_buffer_pool_in_core_dump option to trim buffer pool from core file +# + +--source include/not_windows.inc +--source include/linux.inc +--source include/have_innodb.inc +--source include/not_valgrind.inc +--source include/not_embedded.inc + +# Disable test on sanitizer builds (MSAN and ASAN) because sanitizers +# introduce additional VMAs marked with the `dd` (VM_DONTDUMP) flag +# in /proc//smaps, which breaks the assumptions in +# include/check_core_dump_trim.inc. +--source include/not_msan.inc +--source include/not_asan.inc + +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; + +--echo # +--echo # Confirm innodb_buffer_pool_in_core_dump works +--echo # +SELECT @@global.innodb_buffer_pool_in_core_dump; +--source include/check_core_dump_trim.inc + +--echo # +--echo # Confirm dynamicity of innodb_buffer_pool_in_core_dump +--echo # +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +--source include/check_core_dump_trim.inc + +--echo # +--echo # Confirm innodb_pool_in_core_dump works even +--echo # after resizing of buffer pool. +--echo # + +set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size; + +set global innodb_buffer_pool_size = 64*1024*1024; +--source include/check_core_dump_trim.inc + +set global innodb_buffer_pool_size = 96*1024*1024; +--source include/check_core_dump_trim.inc + +set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size; +set global innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; +--echo # End of 10.11 tests diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result new file mode 100644 index 0000000000000..99d64e7db0e75 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result @@ -0,0 +1,24 @@ +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; +SELECT @@global.innodb_buffer_pool_in_core_dump; +@@global.innodb_buffer_pool_in_core_dump +0 +SELECT @@session.innodb_buffer_pool_in_core_dump; +ERROR HY000: Variable 'innodb_buffer_pool_in_core_dump' is a GLOBAL variable +SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +Variable_name Value +innodb_buffer_pool_in_core_dump OFF +SHOW SESSION VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +Variable_name Value +innodb_buffer_pool_in_core_dump OFF +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_BUFFER_POOL_IN_CORE_DUMP OFF +SELECT * FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_BUFFER_POOL_IN_CORE_DUMP OFF +SET SESSION innodb_buffer_pool_in_core_dump=ON; +ERROR HY000: Variable 'innodb_buffer_pool_in_core_dump' is a GLOBAL variable and should be set with SET GLOBAL +SET GLOBAL innodb_buffer_pool_in_core_dump=ON; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; +SET GLOBAL innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 51143f6686927..ecdcb492f82dc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -5,6 +5,7 @@ variable_name not in ( 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS 'innodb_buffer_pool_size_max', # default value depends on OS +'innodb_buffer_pool_in_core_dump', # only available on Linux and FreeBSD 'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_linux_aio', # existence depends on OS 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test new file mode 100644 index 0000000000000..b4daf005343b3 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test @@ -0,0 +1,31 @@ +# +# Basic test for innodb_buffer_pool_in_core_dump +# + +--source include/have_innodb.inc +if (`SELECT COUNT(*) = 0 from INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE LOWER(variable_name) ='innodb_buffer_pool_in_core_dump'`) +{ + --skip Test requires innodb_buffer_pool_in_core_dump variable. +} + +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; + +SELECT @@global.innodb_buffer_pool_in_core_dump; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.innodb_buffer_pool_in_core_dump; + +SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +SHOW SESSION VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; + +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +SELECT * FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; + +--error ER_GLOBAL_VARIABLE +SET SESSION innodb_buffer_pool_in_core_dump=ON; + +SET GLOBAL innodb_buffer_pool_in_core_dump=ON; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; + +SET GLOBAL innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test index a47691a7af685..7cda33d9fb612 100644 --- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test +++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test @@ -16,6 +16,7 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS 'innodb_buffer_pool_size_max', # default value depends on OS + 'innodb_buffer_pool_in_core_dump', # only available on Linux and FreeBSD 'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_linux_aio', # existence depends on OS 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 4e8bb0894dabd..cb7b12ed0d32e 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1310,6 +1310,12 @@ bool buf_pool_t::create() noexcept allocated before innodb initialization */ ut_ad(srv_operation >= SRV_OPERATION_RESTORE || !field_ref_zero); +#if defined(__aarch64__) + mysql_mutex_init(buf_pool_mutex_key, &mutex, MY_MUTEX_INIT_FAST); +#else + mysql_mutex_init(buf_pool_mutex_key, &mutex, nullptr); +#endif + if (!field_ref_zero) { if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) @@ -1405,7 +1411,11 @@ bool buf_pool_t::create() noexcept memory_unaligned= nullptr; goto oom; } - ut_dontdump(memory_unaligned, size_unaligned, true); +#if defined __linux__ || defined __FreeBSD__ + ut_d(mysql_mutex_lock(&mutex)); + core_advise(); + ut_d(mysql_mutex_unlock(&mutex)); +#endif #else update_malloc_size(actual_size, 0); #endif @@ -1451,12 +1461,6 @@ bool buf_pool_t::create() noexcept } } -#if defined(__aarch64__) - mysql_mutex_init(buf_pool_mutex_key, &mutex, MY_MUTEX_INIT_FAST); -#else - mysql_mutex_init(buf_pool_mutex_key, &mutex, nullptr); -#endif - UT_LIST_INIT(withdrawn, &buf_page_t::list); UT_LIST_INIT(LRU, &buf_page_t::LRU); UT_LIST_INIT(flush_list, &buf_page_t::list); @@ -1962,9 +1966,11 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept return; } - ut_dontdump(memory + old_size, size - old_size, true); size_in_bytes_requested= size; size_in_bytes= size; +#if defined __linux__ || defined __FreeBSD__ + core_advise(); +#endif { const size_t ssize= srv_page_size_shift - UNIV_PAGE_SIZE_SHIFT_MIN; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index abffb5f5a10e3..df5c3d80395cb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18770,6 +18770,19 @@ innodb_encrypt_tables_update(THD*, st_mysql_sys_var*, void*, const void* save) mysql_mutex_lock(&LOCK_global_system_variables); } +#if defined __linux__ || defined __FreeBSD__ +/** Update the system variable innodb_buffer_pool_in_core_dump. +@param save to-be-assigned value */ +static void innodb_buffer_pool_in_core_dump_update(THD *, st_mysql_sys_var *, + void *, const void *save) +{ + mysql_mutex_lock(&buf_pool.mutex); + buf_pool.in_core_dump= *static_cast(save); + buf_pool.core_advise(); + mysql_mutex_unlock(&buf_pool.mutex); +} +#endif + static SHOW_VAR innodb_status_variables_export[]= { SHOW_FUNC_ENTRY("Innodb", &show_innodb_vars), {NullS, NullS, SHOW_LONG} @@ -19284,6 +19297,12 @@ static MYSQL_SYSVAR_BOOL(buffer_pool_dump_at_shutdown, srv_buffer_pool_dump_at_s "Dump the buffer pool into a file named @@innodb_buffer_pool_filename", NULL, NULL, TRUE); +#if defined __linux__ || defined __FreeBSD__ +static MYSQL_SYSVAR_BOOL(buffer_pool_in_core_dump, buf_pool.in_core_dump, + PLUGIN_VAR_OPCMDARG, "Include the buffer pool in core dump.", + nullptr, innodb_buffer_pool_in_core_dump_update, IF_DBUG(true,false)); +#endif + static MYSQL_SYSVAR_ULONG(buffer_pool_dump_pct, srv_buf_pool_dump_pct, PLUGIN_VAR_RQCMDARG, "Dump only the hottest N% of each buffer pool, defaults to 25", @@ -19978,6 +19997,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(buffer_pool_filename), MYSQL_SYSVAR(buffer_pool_dump_now), MYSQL_SYSVAR(buffer_pool_dump_at_shutdown), +#if defined __linux__ || defined __FreeBSD__ + MYSQL_SYSVAR(buffer_pool_in_core_dump), +#endif MYSQL_SYSVAR(buffer_pool_dump_pct), #ifdef UNIV_DEBUG MYSQL_SYSVAR(buffer_pool_evict), diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3213af1518a4f..805390958e2af 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1250,6 +1250,15 @@ class buf_pool_t static int madvise_do_dump() noexcept; #endif +#if defined __linux__ || defined __FreeBSD__ + /** Include or exclude the buffer pool from core dump. */ + void core_advise() noexcept + { + mysql_mutex_assert_owner(&mutex); + madvise(memory, size_in_bytes, in_core_dump ? MADV_DODUMP : MADV_DONTDUMP); + } +#endif + /** Hash cell chain in page_hash_table */ struct hash_chain { @@ -1829,6 +1838,13 @@ class buf_pool_t set while holding mutex, cleared while holding flush_list_mutex */ Atomic_relaxed LRU_warned; +#if defined __linux__ || defined __FreeBSD__ +public: + /** The value of innodb_buffer_pool_in_core_dump */ + my_bool in_core_dump; +private: +#endif + /** withdrawn blocks during resize() */ UT_LIST_BASE_NODE_T(buf_page_t) withdrawn;