From 8bb4f05db5151b8d5c7ca264424787eeb059bb20 Mon Sep 17 00:00:00 2001 From: Mohammad Tafzeel Shams Date: Sat, 14 Feb 2026 17:45:53 +0530 Subject: [PATCH] MDEV-22186: Add innodb_buffer_pool_in_core_dump to exclude InnoDB buffer pool from cores Problem: There is no control to include or exclude large InnoDB buffer pool from core files, which can lead to unnecessarily large core dumps or prevent capturing useful memory state. Solution: Introduce a new global dynamic system variable innodb_buffer_pool_in_core_dump that allows excluding InnoDB buffer pool from core dumps using MADV_DONTDUMP where supported. When enabled the server marks relevant memory regions to be omitted from core files. The madvise state is dynamically updated when the variable changes. This variable is only available on Linux & FreeBSD. Additionally, a test helper script is introduced to inspect /proc//smaps and verify whether large memory regions are marked for trimming. - innodb_buffer_pool_in_core_dump Global boolean system variable controlling whether InnoDB buffer pool should be excluded from core dumps. Default: OFF on non-debug builds with MADV_DONTDUMP support, ON otherwise. - innodb_buffer_pool_in_core_dump_update() Update hook that sets innodb_buffer_pool_in_core_dump and triggers runtime updates of madvise state for the buffer pool. - buf_pool_t::in_core_dump Variable backing @@innodb_buffer_pool_in_core_dump. - buf_pool_t::core_advise() Advises MADV_DONTDUMP or MADV_DODUMP according to latest innodb_buffer_pool_in_core_dump value. - buf_pool_t::create() Updates madvise flag to the buffer pool memory based on innodb_buffer_pool_in_core_dump. Also moves the mutex initialization at the beginiing of function. - buf_pool_t::resize() Updates the madvise flag for buffer pool to keep it in sync with the innodb_buffer_pool_in_core_dump. - check_core_dump_trim.inc Test helper that inspects /proc//smaps to detect memory regions with the dd VmFlag and determine whether the core file is expected to be trimmed based on buffer pool and log buffer sizes. --- .../innodb/include/check_core_dump_trim.inc | 104 ++++++++++++++++++ .../innodb/r/buffer_pool_in_core_dump.result | 31 ++++++ .../innodb/t/buffer_pool_in_core_dump.opt | 2 + .../innodb/t/buffer_pool_in_core_dump.test | 56 ++++++++++ ...nodb_buffer_pool_in_core_dump_basic.result | 24 ++++ .../suite/sys_vars/r/sysvars_innodb.result | 1 + ...innodb_buffer_pool_in_core_dump_basic.test | 31 ++++++ .../suite/sys_vars/t/sysvars_innodb.test | 1 + storage/innobase/buf/buf0buf.cc | 22 ++-- storage/innobase/handler/ha_innodb.cc | 22 ++++ storage/innobase/include/buf0buf.h | 16 +++ 11 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/innodb/include/check_core_dump_trim.inc create mode 100644 mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result create mode 100644 mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt create mode 100644 mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test create mode 100644 mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result create mode 100644 mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test diff --git a/mysql-test/suite/innodb/include/check_core_dump_trim.inc b/mysql-test/suite/innodb/include/check_core_dump_trim.inc new file mode 100644 index 0000000000000..7f3ebec0f5b65 --- /dev/null +++ b/mysql-test/suite/innodb/include/check_core_dump_trim.inc @@ -0,0 +1,104 @@ +# ------------------------------------------------------------------------------ +# Test Summary +# ------------------------------------------------------------------------------ +# This test checks whether the mariadbd core dump file is expected to be TRIMMED +# or NOT TRIMMED based on how memory is mapped by the mariadbd process. +# +# What this script does: +# +# 1. It retrieves MariaDB runtime values: +# - pid_file (location of mariadbd PID) +# - innodb_buffer_pool_size +# +# 2. Using the PID file, it finds the running mariadbd process ID. +# +# 3. It reads /proc//smaps to inspect the memory regions allocated +# by the mariadbd process. +# +# 4. From the smaps output, it identifies candidate memory regions that: +# - are smaller than 192M +# - contain the "dd" VmFlag (indicating the region is eligible for +# core dump trimming) +# +# 5. It sums the size of these candidate memory regions. +# +# 6. If the candidate memory is large enough to cover the buffer pool, +# the script concludes that the buffer pool will not be part of core dump. +# Otherwise, it reports it to be part of core dump. +# +# Purpose: +# This helps verify whether InnoDB buffer pool will be excluded from the +# core dump, preventing unnecessarily large core files. +# ------------------------------------------------------------------------------ + +# Get the full path name of the PID file +--let $pid_file= query_get_value(SELECT @@pid_file, @@pid_file, 1) +--let PIDFILE= $pid_file + +# Get innodb_buffer_pool_size +--let $innodb_buffer_pool_size= query_get_value(SELECT @@innodb_buffer_pool_size, @@innodb_buffer_pool_size, 1) +--let BUFFER_POOL_SIZE= $innodb_buffer_pool_size + +perl; + +use strict; +use warnings; + +my $pid_file = $ENV{'PIDFILE'} or die "PIDFILE not set"; + +# Buffer pool size +my $buffer_pool_size = $ENV{'BUFFER_POOL_SIZE'}; + +# Convert to KB +my $buffer_pool_kb = int($buffer_pool_size / 1024); + +# Ignore regions larger than 192M +my $max_region_kb = 192 * 1024; + +# Sum of candidate regions +my $candidate_sum_kb = 0; + +# Get PID of mariadbd +open(my $fh, '<', $pid_file) || die "Cannot open pid file $pid_file\n"; +my $pid = <$fh>; +$pid =~ s/\s//g; +close($fh); + +if ($pid eq "") { + die "Couldn't retrieve PID from PID file.\n"; +} + +# Open smaps +my $smaps_file = "/proc/$pid/smaps"; + +open(my $sm, '<', $smaps_file) or die "Cannot open $smaps_file\n"; + +my $current_size_kb = 0; + +while (my $line = <$sm>) { + + if ($line =~ /^Size:\s+(\d+)\s+kB/) { + $current_size_kb = $1; + } + + if ($line =~ /^VmFlags:\s+(.*)/) { + my $flags = $1; + + # Candidate region conditions + if ($current_size_kb <= $max_region_kb && + $flags =~ /\bdd\b/) { + $candidate_sum_kb += $current_size_kb; + } + } +} + +close($sm); + +if ($candidate_sum_kb >= $buffer_pool_kb) { + print "Buffer pool not part of core dump.\n"; +} +else { + print "Buffer pool part of core dump.\n"; +} + +EOF diff --git a/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result b/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result new file mode 100644 index 0000000000000..ac64260b85cad --- /dev/null +++ b/mysql-test/suite/innodb/r/buffer_pool_in_core_dump.result @@ -0,0 +1,31 @@ +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +# +# Confirm innodb_buffer_pool_in_core_dump works +# +SELECT @@global.innodb_buffer_pool_in_core_dump; +@@global.innodb_buffer_pool_in_core_dump +0 +Buffer pool not part of core dump. +# +# Confirm dynamicity of innodb_buffer_pool_in_core_dump +# +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +Buffer pool part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +Buffer pool not part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +Buffer pool part of core dump. +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +Buffer pool not part of core dump. +# +# Confirm innodb_pool_in_core_dump works even +# after resizing of buffer pool. +# +set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size; +set global innodb_buffer_pool_size = 64*1024*1024; +Buffer pool not part of core dump. +set global innodb_buffer_pool_size = 96*1024*1024; +Buffer pool not part of core dump. +set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size; +set global innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; +# End of 10.11 tests diff --git a/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt new file mode 100644 index 0000000000000..c388fc9c426b4 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.opt @@ -0,0 +1,2 @@ +--loose-skip-innodb-buffer-pool-in-core-dump +--innodb-buffer-pool-size=128M diff --git a/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test new file mode 100644 index 0000000000000..e22c1ed7aed19 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_in_core_dump.test @@ -0,0 +1,56 @@ +# +# MDEV-22186 : Add innodb_buffer_pool_in_core_dump option to trim buffer pool from core file +# + +--source include/not_windows.inc +--source include/linux.inc +--source include/have_innodb.inc +--source include/not_valgrind.inc +--source include/not_embedded.inc + +# Disable test on sanitizer builds (MSAN and ASAN) because sanitizers +# introduce additional VMAs marked with the `dd` (VM_DONTDUMP) flag +# in /proc//smaps, which breaks the assumptions in +# include/check_core_dump_trim.inc. +--source include/not_msan.inc +--source include/not_asan.inc + +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; + +--echo # +--echo # Confirm innodb_buffer_pool_in_core_dump works +--echo # +SELECT @@global.innodb_buffer_pool_in_core_dump; +--source include/check_core_dump_trim.inc + +--echo # +--echo # Confirm dynamicity of innodb_buffer_pool_in_core_dump +--echo # +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = ON; +--source include/check_core_dump_trim.inc + +SET GLOBAL innodb_buffer_pool_in_core_dump = OFF; +--source include/check_core_dump_trim.inc + +--echo # +--echo # Confirm innodb_pool_in_core_dump works even +--echo # after resizing of buffer pool. +--echo # + +set @old_innodb_buffer_pool_size = @@innodb_buffer_pool_size; + +set global innodb_buffer_pool_size = 64*1024*1024; +--source include/check_core_dump_trim.inc + +set global innodb_buffer_pool_size = 96*1024*1024; +--source include/check_core_dump_trim.inc + +set global innodb_buffer_pool_size = @old_innodb_buffer_pool_size; +set global innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; +--echo # End of 10.11 tests diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result new file mode 100644 index 0000000000000..99d64e7db0e75 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_in_core_dump_basic.result @@ -0,0 +1,24 @@ +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; +SELECT @@global.innodb_buffer_pool_in_core_dump; +@@global.innodb_buffer_pool_in_core_dump +0 +SELECT @@session.innodb_buffer_pool_in_core_dump; +ERROR HY000: Variable 'innodb_buffer_pool_in_core_dump' is a GLOBAL variable +SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +Variable_name Value +innodb_buffer_pool_in_core_dump OFF +SHOW SESSION VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +Variable_name Value +innodb_buffer_pool_in_core_dump OFF +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_BUFFER_POOL_IN_CORE_DUMP OFF +SELECT * FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +VARIABLE_NAME VARIABLE_VALUE +INNODB_BUFFER_POOL_IN_CORE_DUMP OFF +SET SESSION innodb_buffer_pool_in_core_dump=ON; +ERROR HY000: Variable 'innodb_buffer_pool_in_core_dump' is a GLOBAL variable and should be set with SET GLOBAL +SET GLOBAL innodb_buffer_pool_in_core_dump=ON; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; +SET GLOBAL innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 51143f6686927..ecdcb492f82dc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -5,6 +5,7 @@ variable_name not in ( 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS 'innodb_buffer_pool_size_max', # default value depends on OS +'innodb_buffer_pool_in_core_dump', # only available on Linux and FreeBSD 'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_linux_aio', # existence depends on OS 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test new file mode 100644 index 0000000000000..b4daf005343b3 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_in_core_dump_basic.test @@ -0,0 +1,31 @@ +# +# Basic test for innodb_buffer_pool_in_core_dump +# + +--source include/have_innodb.inc +if (`SELECT COUNT(*) = 0 from INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE LOWER(variable_name) ='innodb_buffer_pool_in_core_dump'`) +{ + --skip Test requires innodb_buffer_pool_in_core_dump variable. +} + +set @old_innodb_buffer_pool_in_core_dump = @@innodb_buffer_pool_in_core_dump; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; + +SELECT @@global.innodb_buffer_pool_in_core_dump; + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.innodb_buffer_pool_in_core_dump; + +SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; +SHOW SESSION VARIABLES LIKE 'innodb_buffer_pool_in_core_dump'; + +SELECT * FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; +SELECT * FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE variable_name='innodb_buffer_pool_in_core_dump'; + +--error ER_GLOBAL_VARIABLE +SET SESSION innodb_buffer_pool_in_core_dump=ON; + +SET GLOBAL innodb_buffer_pool_in_core_dump=ON; +SET GLOBAL innodb_buffer_pool_in_core_dump=OFF; + +SET GLOBAL innodb_buffer_pool_in_core_dump = @old_innodb_buffer_pool_in_core_dump; diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test index a47691a7af685..7cda33d9fb612 100644 --- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test +++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test @@ -16,6 +16,7 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP 'innodb_evict_tables_on_commit_debug', # one may want to override this 'innodb_use_native_aio', # default value depends on OS 'innodb_buffer_pool_size_max', # default value depends on OS + 'innodb_buffer_pool_in_core_dump', # only available on Linux and FreeBSD 'innodb_log_file_buffering', # only available on Linux and Windows 'innodb_linux_aio', # existence depends on OS 'innodb_buffer_pool_load_pages_abort') # debug build only, and is only for testing diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 4e8bb0894dabd..cb7b12ed0d32e 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1310,6 +1310,12 @@ bool buf_pool_t::create() noexcept allocated before innodb initialization */ ut_ad(srv_operation >= SRV_OPERATION_RESTORE || !field_ref_zero); +#if defined(__aarch64__) + mysql_mutex_init(buf_pool_mutex_key, &mutex, MY_MUTEX_INIT_FAST); +#else + mysql_mutex_init(buf_pool_mutex_key, &mutex, nullptr); +#endif + if (!field_ref_zero) { if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) @@ -1405,7 +1411,11 @@ bool buf_pool_t::create() noexcept memory_unaligned= nullptr; goto oom; } - ut_dontdump(memory_unaligned, size_unaligned, true); +#if defined __linux__ || defined __FreeBSD__ + ut_d(mysql_mutex_lock(&mutex)); + core_advise(); + ut_d(mysql_mutex_unlock(&mutex)); +#endif #else update_malloc_size(actual_size, 0); #endif @@ -1451,12 +1461,6 @@ bool buf_pool_t::create() noexcept } } -#if defined(__aarch64__) - mysql_mutex_init(buf_pool_mutex_key, &mutex, MY_MUTEX_INIT_FAST); -#else - mysql_mutex_init(buf_pool_mutex_key, &mutex, nullptr); -#endif - UT_LIST_INIT(withdrawn, &buf_page_t::list); UT_LIST_INIT(LRU, &buf_page_t::LRU); UT_LIST_INIT(flush_list, &buf_page_t::list); @@ -1962,9 +1966,11 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept return; } - ut_dontdump(memory + old_size, size - old_size, true); size_in_bytes_requested= size; size_in_bytes= size; +#if defined __linux__ || defined __FreeBSD__ + core_advise(); +#endif { const size_t ssize= srv_page_size_shift - UNIV_PAGE_SIZE_SHIFT_MIN; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index abffb5f5a10e3..df5c3d80395cb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18770,6 +18770,19 @@ innodb_encrypt_tables_update(THD*, st_mysql_sys_var*, void*, const void* save) mysql_mutex_lock(&LOCK_global_system_variables); } +#if defined __linux__ || defined __FreeBSD__ +/** Update the system variable innodb_buffer_pool_in_core_dump. +@param save to-be-assigned value */ +static void innodb_buffer_pool_in_core_dump_update(THD *, st_mysql_sys_var *, + void *, const void *save) +{ + mysql_mutex_lock(&buf_pool.mutex); + buf_pool.in_core_dump= *static_cast(save); + buf_pool.core_advise(); + mysql_mutex_unlock(&buf_pool.mutex); +} +#endif + static SHOW_VAR innodb_status_variables_export[]= { SHOW_FUNC_ENTRY("Innodb", &show_innodb_vars), {NullS, NullS, SHOW_LONG} @@ -19284,6 +19297,12 @@ static MYSQL_SYSVAR_BOOL(buffer_pool_dump_at_shutdown, srv_buffer_pool_dump_at_s "Dump the buffer pool into a file named @@innodb_buffer_pool_filename", NULL, NULL, TRUE); +#if defined __linux__ || defined __FreeBSD__ +static MYSQL_SYSVAR_BOOL(buffer_pool_in_core_dump, buf_pool.in_core_dump, + PLUGIN_VAR_OPCMDARG, "Include the buffer pool in core dump.", + nullptr, innodb_buffer_pool_in_core_dump_update, IF_DBUG(true,false)); +#endif + static MYSQL_SYSVAR_ULONG(buffer_pool_dump_pct, srv_buf_pool_dump_pct, PLUGIN_VAR_RQCMDARG, "Dump only the hottest N% of each buffer pool, defaults to 25", @@ -19978,6 +19997,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(buffer_pool_filename), MYSQL_SYSVAR(buffer_pool_dump_now), MYSQL_SYSVAR(buffer_pool_dump_at_shutdown), +#if defined __linux__ || defined __FreeBSD__ + MYSQL_SYSVAR(buffer_pool_in_core_dump), +#endif MYSQL_SYSVAR(buffer_pool_dump_pct), #ifdef UNIV_DEBUG MYSQL_SYSVAR(buffer_pool_evict), diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3213af1518a4f..805390958e2af 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1250,6 +1250,15 @@ class buf_pool_t static int madvise_do_dump() noexcept; #endif +#if defined __linux__ || defined __FreeBSD__ + /** Include or exclude the buffer pool from core dump. */ + void core_advise() noexcept + { + mysql_mutex_assert_owner(&mutex); + madvise(memory, size_in_bytes, in_core_dump ? MADV_DODUMP : MADV_DONTDUMP); + } +#endif + /** Hash cell chain in page_hash_table */ struct hash_chain { @@ -1829,6 +1838,13 @@ class buf_pool_t set while holding mutex, cleared while holding flush_list_mutex */ Atomic_relaxed LRU_warned; +#if defined __linux__ || defined __FreeBSD__ +public: + /** The value of innodb_buffer_pool_in_core_dump */ + my_bool in_core_dump; +private: +#endif + /** withdrawn blocks during resize() */ UT_LIST_BASE_NODE_T(buf_page_t) withdrawn;