From 900bbbe4a89cf7fd2f443bc13ce6af2291f8ac21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:11:43 +0200 Subject: [PATCH 1/3] MDEV-33295 innodb.doublewrite occasionally fails When the first attempt of XA ROLLBACK is expected to fail, some recovered changes could be written back through the doublewrite buffer. Should that happen, the next recovery attempt (after mangling the data file t1.ibd further) could fail because no copy of the affected pages would be available in the doublewrite buffer. To prevent this from happening, ensure that the doublewrite buffer will not be used and no log checkpoint occurs during the previous failed recovery attempt. Also, let a successful XA ROLLBACK serve the additional purpose of freeing a BLOB page and therefore rewriting page 0, which we must then be able to recover despite induced corruption. In the last restart step, we will tolerate an unexpected checkpoint, because one is frequently occurring on FreeBSD and AIX, despite our efforts to force a buffer pool flush before each "no checkpoint" section. --- mysql-test/suite/innodb/r/doublewrite.result | 18 +++----- .../suite/innodb/t/doublewrite.combinations | 2 + mysql-test/suite/innodb/t/doublewrite.test | 46 ++++++++++++------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 667c65d7b7ba4..d381c6850c34a 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -11,9 +11,11 @@ insert into t1 values(5, repeat('.',12)); commit work; SET GLOBAL innodb_fast_shutdown = 0; # restart +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +SET GLOBAL innodb_max_dirty_pages_pct=99; connect dml,localhost,root,,; XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +insert into t1 values(6, repeat('%', @@innodb_page_size/2)); XA END 'x'; XA PREPARE 'x'; disconnect dml; @@ -23,7 +25,6 @@ flush table t1 for export; # restart FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=0\]/ in mysqld.1.err # restart -XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text test.t1 check status OK @@ -34,18 +35,13 @@ f1 f2 3 //////////// 4 ------------ 5 ............ -connect dml,localhost,root,,; -XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); -XA END 'x'; -XA PREPARE 'x'; -disconnect dml; -connection default; -flush table t1 for export; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +SET GLOBAL innodb_max_dirty_pages_pct=99; +XA ROLLBACK 'x'; +FLUSH TABLE t1 FOR EXPORT; # Kill the server # restart FOUND 4 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=[03]\]/ in mysqld.1.err -XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/doublewrite.combinations b/mysql-test/suite/innodb/t/doublewrite.combinations index 4f52013f6fc37..797b715877888 100644 --- a/mysql-test/suite/innodb/t/doublewrite.combinations +++ b/mysql-test/suite/innodb/t/doublewrite.combinations @@ -1,7 +1,9 @@ [strict_crc32] --innodb-checksum-algorithm=strict_crc32 --innodb-use-atomic-writes=0 +--innodb-undo-tablespaces=0 [strict_full_crc32] --innodb-checksum-algorithm=strict_full_crc32 --innodb-use-atomic-writes=0 +--innodb-undo-tablespaces=0 diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index faafbb7cbac53..a9c81901c4723 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -42,13 +42,17 @@ commit work; SET GLOBAL innodb_fast_shutdown = 0; let $shutdown_timeout=; --source include/restart_mysqld.inc -# Ensure that buf_flush_page_cleaner() has woken up from its -# first my_cond_timedwait() and gone idle. -sleep 1; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +let $wait_condition = +SELECT variable_value = 0 +FROM information_schema.global_status +WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +--source include/wait_condition.inc +SET GLOBAL innodb_max_dirty_pages_pct=99; --source ../include/no_checkpoint_start.inc connect (dml,localhost,root,,); XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +insert into t1 values(6, repeat('%', @@innodb_page_size/2)); XA END 'x'; XA PREPARE 'x'; disconnect dml; @@ -56,10 +60,12 @@ connection default; flush table t1 for export; -let $restart_parameters=; --let CLEANUP_IF_CHECKPOINT=drop table t1, unexpected_checkpoint; --source ../include/no_checkpoint_end.inc +--copy_file $MYSQLD_DATADIR/ibdata1 $MYSQLD_DATADIR/ibdata1.bak +--copy_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile0.bak + perl; use IO::Handle; do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; @@ -148,6 +154,12 @@ let $shutdown_timeout=0; --source include/shutdown_mysqld.inc let $shutdown_timeout=; # Corrupt the file in a better way. + +--remove_file $MYSQLD_DATADIR/ibdata1 +--remove_file $MYSQLD_DATADIR/ib_logfile0 +--move_file $MYSQLD_DATADIR/ibdata1.bak $MYSQLD_DATADIR/ibdata1 +--move_file $MYSQLD_DATADIR/ib_logfile0.bak $MYSQLD_DATADIR/ib_logfile0 + perl; use IO::Handle; my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; @@ -160,22 +172,23 @@ syswrite(FILE, chr(0) x ($page_size/2)); close FILE; EOF --source include/start_mysqld.inc -XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +let $wait_condition = +SELECT variable_value = 0 +FROM information_schema.global_status +WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +--source include/wait_condition.inc +SET GLOBAL innodb_max_dirty_pages_pct=99; --source ../include/no_checkpoint_start.inc -connect (dml,localhost,root,,); -XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); -XA END 'x'; -XA PREPARE 'x'; -disconnect dml; -connection default; - -flush table t1 for export; +XA ROLLBACK 'x'; +FLUSH TABLE t1 FOR EXPORT; -let $restart_parameters=; +# If we are skipping the test at this point due to an unexpected +# checkpoint, we will already have tested a part of this functionality. +--let CLEANUP_IF_CHECKPOINT=drop table t1; --source ../include/no_checkpoint_end.inc # Zero out the first page in file and try to recover from dblwr @@ -189,7 +202,6 @@ EOF --source include/start_mysqld.inc let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=[03]\\]; --source include/search_pattern_in_file.inc -XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; drop table t1; From f6fd591a0896b54bc32409486420778d05947a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:28:01 +0200 Subject: [PATCH 2/3] mtr: Globally suppress some rare warnings But do not suppress InnoDB errors that should be impossible starting with MariaDB Server 10.6. --- mysql-test/mariadb-test-run.pl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 3cfd2db2e03f2..a47be67a05026 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -4458,14 +4458,12 @@ ($$) ( @global_suppressions, qr/error .*connecting to master/, - qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/, - qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/, - qr/InnoDB: Warning: a long semaphore wait:/, qr/InnoDB: Dumping buffer pool.*/, qr/InnoDB: Buffer pool.*/, qr/InnoDB: Could not free any blocks in the buffer pool!/, - qr/InnoDB: Warning: Writer thread is waiting this semaphore:/, qr/InnoDB: innodb_open_files .* should not be greater than/, + qr/InnoDB: Trying to delete tablespace.*but there are.*pending/, + qr/InnoDB: Tablespace 1[0-9]* was not found at .*, and innodb_force_recovery was set/, qr/Slave: Unknown table 't1' .* 1051/, qr/Slave SQL:.*(Internal MariaDB error code: [[:digit:]]+|Query:.*)/, qr/slave SQL thread aborted/, From 75b24a002ff5b95f2d6ae28231227c2588e7d8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:29:52 +0200 Subject: [PATCH 3/3] Suppress processist_state='buffer pool load' Sometimes, thread/innodb/thread_pool_thread may display processlist_state='buffer pool load' instead of NULL when the test is executed soon enough after server startup. Let us suppress that information to avoid spurious failures. --- mysql-test/suite/perfschema/r/threads_innodb.result | 10 +++++----- mysql-test/suite/perfschema/t/threads_innodb.test | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/perfschema/r/threads_innodb.result b/mysql-test/suite/perfschema/r/threads_innodb.result index abfedb57644c4..bd81c5991f32d 100644 --- a/mysql-test/suite/perfschema/r/threads_innodb.result +++ b/mysql-test/suite/perfschema/r/threads_innodb.result @@ -1,10 +1,10 @@ SELECT name, type, processlist_user, processlist_host, processlist_db, -processlist_command, processlist_time, processlist_state, processlist_info, +processlist_command, processlist_time, processlist_info, parent_thread_id, role, instrumented FROM performance_schema.threads WHERE name LIKE 'thread/innodb/%' GROUP BY name; -name type processlist_user processlist_host processlist_db processlist_command processlist_time processlist_state processlist_info parent_thread_id role instrumented -thread/innodb/page_cleaner_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES -thread/innodb/page_encrypt_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES -thread/innodb/thread_pool_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES +name type processlist_user processlist_host processlist_db processlist_command processlist_time processlist_info parent_thread_id role instrumented +thread/innodb/page_cleaner_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES +thread/innodb/page_encrypt_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES +thread/innodb/thread_pool_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES diff --git a/mysql-test/suite/perfschema/t/threads_innodb.test b/mysql-test/suite/perfschema/t/threads_innodb.test index 366c3b82ce82e..e791ded2597f8 100644 --- a/mysql-test/suite/perfschema/t/threads_innodb.test +++ b/mysql-test/suite/perfschema/t/threads_innodb.test @@ -14,7 +14,7 @@ # We suppress here duplicates rows with the goal to avoid that the test fails # in case some defaults are changed. SELECT name, type, processlist_user, processlist_host, processlist_db, - processlist_command, processlist_time, processlist_state, processlist_info, + processlist_command, processlist_time, processlist_info, parent_thread_id, role, instrumented FROM performance_schema.threads WHERE name LIKE 'thread/innodb/%'