From 68dfc8561cf6c50eb697bca2413f8606e781a826 Mon Sep 17 00:00:00 2001 From: Kamil Holubicki Date: Fri, 22 May 2026 01:40:50 +0200 Subject: [PATCH] PS-11080 fix: Spoiled logical clock information in rewritten binlog (part 4) https://perconadev.atlassian.net/browse/PS-11080 Added MTR test cases for the 'sequence_number' / 'last_committed' field rewrite logic: * 'gtid_renumbering' - checks for renumbering when remote rotation occurs. * 'gtid_renumbering_local_rotation' - checks for renumbering when local rotation occurs. * 'gtid_renumbering_resume' - checks that 'last_sequence_number' is restored properly from the binlog metadata file upon PBS restart. * 'gtid_renumbering_resume_after_partial' - checks that 'last_sequence_number' in the binlog metadata file always corresponds to the actual data file (not the last seen value currently in the storage buffer). This helps with resuming PBS after the crash. Currently 'gtid_renumbering_resume_after_partial' MTR test case requires DEBUG_SYNC functionality to be available in the MySQL Server. Unfortunately, at the moment we use release tarballs of the MySQL Server in GitHub Actions workers and this test will always be skipped there. Co-authored-by: Yura Sorokin --- .../r/gtid_renumbering.result | 47 ++++ .../r/gtid_renumbering_local_rotation.result | 34 +++ .../r/gtid_renumbering_resume.result | 50 ++++ ...id_renumbering_resume_after_partial.result | 78 ++++++ .../t/gtid_renumbering-master.opt | 2 + mtr/binlog_streaming/t/gtid_renumbering.test | 209 ++++++++++++++++ ...gtid_renumbering_local_rotation-master.opt | 2 + .../t/gtid_renumbering_local_rotation.test | 145 ++++++++++++ .../t/gtid_renumbering_resume-master.opt | 2 + .../t/gtid_renumbering_resume.test | 205 ++++++++++++++++ ...enumbering_resume_after_partial-master.opt | 2 + ...gtid_renumbering_resume_after_partial.test | 224 ++++++++++++++++++ 12 files changed, 1000 insertions(+) create mode 100644 mtr/binlog_streaming/r/gtid_renumbering.result create mode 100644 mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result create mode 100644 mtr/binlog_streaming/r/gtid_renumbering_resume.result create mode 100644 mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result create mode 100644 mtr/binlog_streaming/t/gtid_renumbering-master.opt create mode 100644 mtr/binlog_streaming/t/gtid_renumbering.test create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_resume.test create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt create mode 100644 mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test diff --git a/mtr/binlog_streaming/r/gtid_renumbering.result b/mtr/binlog_streaming/r/gtid_renumbering.result new file mode 100644 index 0000000..4fb6649 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering.result @@ -0,0 +1,47 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Building deterministic binlog content with two source-binlog +*** rotations between transaction groups. + +*** Source binlog A: CREATE TABLE + 9 INSERTs (10 GTIDs total). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** Flushing source binary log (introduces the first +*** source-binlog rotation that the rewriter must absorb). +FLUSH BINARY LOGS; + +*** Source binlog B: 5 INSERTs. + +*** Flushing source binary log one more time (second +*** source-binlog rotation absorbed inside the same local file). +FLUSH BINARY LOGS; + +*** Source binlog C: 3 INSERTs. + +*** Executing the Binlog Server utility in rewrite mode. + +*** Materializing the local binlog file produced by the rewriter +*** and dumping it via mysqlbinlog for textual inspection. + +*** Validating (sequence_number, last_committed) of every GTID +*** event in the local binlog file. Aborts with --die on any +*** invariant violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result b/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result new file mode 100644 index 0000000..abd2418 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result @@ -0,0 +1,34 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Building a single-source-binlog workload that is large enough +*** to span multiple 1K local binlog files after rewrite. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, payload VARCHAR(100) NOT NULL, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** Executing the Binlog Server utility in rewrite mode. + +*** Asserting that the rewriter actually rotated locally +*** (bnlg.000002 must exist - otherwise the test does not +*** exercise the local-rotation reset code path). + +*** Validating per-local-file sequence_number reset and +*** last_committed range. Aborts with --die on violation; +*** produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_resume.result b/mtr/binlog_streaming/r/gtid_renumbering_resume.result new file mode 100644 index 0000000..107c847 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_resume.result @@ -0,0 +1,50 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Phase 1: CREATE TABLE + 4 INSERTs in source binlog A +*** (5 GTID events). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** First binsrv invocation: writes 5 GTID events with +*** sequence_number 1..5 to bnlg.000001 and persists the +*** renumberer recovery snapshot (next_local_seq=5) in the +*** per-binlog metadata file. + +*** Flushing source binary log between invocations so the +*** second binsrv run also has to absorb a source-side +*** rotation right after resuming. +FLUSH BINARY LOGS; + +*** Phase 2: 5 INSERTs in source binlog B (5 more GTID events). + +*** Second binsrv invocation: must resume the renumberer from +*** the persisted snapshot (next_local_seq=5) and append +*** sequence_number 6..10 to bnlg.000001. + +*** Materializing the local binlog file produced across the two +*** invocations and dumping it via mysqlbinlog for textual +*** inspection. + +*** Validating that sequence_number is gap-free across the +*** binsrv-invocation boundary and that the post-resume +*** source-binlog-boundary event has last_committed = +*** sequence_number - 1. Aborts with --die on any invariant +*** violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result b/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result new file mode 100644 index 0000000..d868470 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result @@ -0,0 +1,78 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Phase 1: T_first = CREATE TABLE t1 (1 GTID event, no +*** Write_rows). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** First binsrv invocation: writes T_first with +*** sequence_number=1 to bnlg.000001 and persists +*** next_local_seq=1 in the per-binlog metadata. + +*** Phase 2: arm the source dump thread to pause AFTER +*** sending the first WRITE_ROWS event of any transaction. +*** Combined with the short binsrv read_timeout, this freezes +*** the second binsrv invocation mid-T_third (after T_third's +*** GTID has been processed by binsrv but before T_third's +*** XID can be received). +SET @old_global_debug = @@global.debug; +SET GLOBAL DEBUG = '+d,dump_thread_wait_after_send_write_rows'; + +*** Workload that the second binsrv invocation will read: +*** T_second (CREATE TABLE t2, no Write_rows; will be +*** absorbed in full) and T_third (INSERT INTO t1, has +*** Write_rows; binsrv will receive its GTID and prefix +*** events and then time out waiting for the rest). +CREATE TABLE t2(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +INSERT INTO t1 VALUES(DEFAULT); + +*** Second binsrv invocation: expected to exit with non-zero +*** status after read_timeout. The storage destructor flushes +*** the COMMITTED transaction (T_second) and persists the +*** per-binlog metadata. The renumberer recovery snapshot +*** persisted here MUST be next_local_seq=2 (committed, in +*** lockstep with the flushed bytes) and not the speculative +*** next_local_seq=3 produced by T_third's GTID rewrite. + +*** Releasing the paused source dump thread: SIGNAL wakes it +*** from the debug_sync wait, after which its first attempt +*** to send the next event fails on the half-closed socket +*** and the dump thread exits. DEBUG flag is then restored so +*** the new dump thread spawned by binsrv 3 does not pause. +SET DEBUG_SYNC = 'now SIGNAL signal.continue'; +SET GLOBAL DEBUG = @old_global_debug; +SET DEBUG_SYNC = 'RESET'; + +*** Third binsrv invocation: the source resends T_third +*** (binsrv never acknowledged it), and the renumberer +*** continues from the persisted committed snapshot. T_third +*** must land as sequence_number=3 so bnlg.000001 ends up +*** gap-free. + +*** Materializing bnlg.000001 produced across all three +*** invocations and dumping it via mysqlbinlog for textual +*** inspection. + +*** Validating that bnlg.000001 contains exactly 3 GTID +*** events with contiguous sequence_numbers 1, 2, 3 and +*** that every last_committed value references an in-file +*** sequence_number. Aborts with --die on any invariant +*** violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the tables. +DROP TABLE t1, t2; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/t/gtid_renumbering-master.opt b/mtr/binlog_streaming/t/gtid_renumbering-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering.test b/mtr/binlog_streaming/t/gtid_renumbering.test new file mode 100644 index 0000000..12b8836 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering.test @@ -0,0 +1,209 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# This test exercises the rewrite-mode GTID renumberer. In rewrite mode +# the binlog server discards the source's ROTATE / FORMAT_DESCRIPTION / +# PREVIOUS_GTIDS / STOP events and coalesces transactions into local +# binlog files of a configurable size. Two consequences: +# 1. Multiple source binlogs may be folded into a single local file. +# The source resets sequence_number to 1 at every rotation, which +# would produce duplicate / non-monotone sequence_number values in +# our local file unless the rewriter renumbers them. +# 2. The first event of each new source-file segment carries +# last_committed=0 from the source, relying on the implicit +# synchronization guarantee that every transaction of the previous +# source file commits before any transaction of the next file +# starts. After coalescing, that physical boundary is gone, so the +# rewriter must replace last_committed=0 with sequence_number-1 to +# preserve the cross-file commit ordering on the consumer side. +# +# We craft three source binlogs (10 + 5 + 3 GTIDs) coalesced into a +# single local file (1G rewrite_file_size) and verify that: +# (a) sequence_number starts at 1 and advances by exactly 1 per event +# (renumbering covers all source-side rotations the rewriter +# absorbed - without the renumberer we would see duplicate +# sequence_number=1 values from each source file); +# (b) last_committed is in [0, sequence_number - 1] for every event; +# (c) the cross-source-file boundary events (events #11 and #16 - +# the first transactions from source binlogs B and C) carry +# a non-zero last_committed (specifically, sequence_number - 1). +# Without the rewriter's segment-boundary fix these would inherit +# the source's verbatim last_committed=0 and let the local +# applier reorder them ahead of the previous source file's +# commits. This assertion is robust regardless of the source's +# binlog_transaction_dependency_tracking mode. + +# in case of --repeat=N, we need to start from a fresh binary log to make +# this test deterministic +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +--let $binsrv_checkpoint_size = 1 +# rewrite_file_size set high enough that all transactions land in a +# single local file - the point of this test is to exercise multiple +# source-file boundaries WITHIN the same local file +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Building deterministic binlog content with two source-binlog +--echo *** rotations between transaction groups. +--echo +--echo *** Source binlog A: CREATE TABLE + 9 INSERTs (10 GTIDs total). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +--disable_query_log +--let $i = 1 +while ($i <= 9) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Flushing source binary log (introduces the first +--echo *** source-binlog rotation that the rewriter must absorb). +FLUSH BINARY LOGS; + +--echo +--echo *** Source binlog B: 5 INSERTs. +--disable_query_log +--let $i = 1 +while ($i <= 5) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Flushing source binary log one more time (second +--echo *** source-binlog rotation absorbed inside the same local file). +FLUSH BINARY LOGS; + +--echo +--echo *** Source binlog C: 3 INSERTs. +--disable_query_log +--let $i = 1 +while ($i <= 3) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Executing the Binlog Server utility in rewrite mode. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing the local binlog file produced by the rewriter +--echo *** and dumping it via mysqlbinlog for textual inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating (sequence_number, last_committed) of every GTID +--echo *** event in the local binlog file. Aborts with --die on any +--echo *** invariant violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # Source A contributes 10 GTIDs (CREATE TABLE + 9 INSERTs), source B + # contributes 5, source C contributes 3 - 18 events in total. + my $expected_count = 18; + die "expected exactly $expected_count GTID events in the local binlog " + . "file, got " . scalar @events + unless @events == $expected_count; + + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + + # Property (a): per-local-file sequence_number must restart at 1 + # and advance gap-free by exactly 1 per event - this is what + # protects us from the source restarting its sequence_number at + # every rotation that we absorb. + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got $events[$i]{sn}" + unless $events[$i]{sn} == $expected_sn; + + # Property (b): last_committed must reference an in-file + # sequence_number, i.e. lie in [0, sequence_number - 1]. A value + # >= sequence_number would point to the future; a negative value + # is impossible on the wire and would indicate a corrupt rewrite. + die "last_committed out of range at sequence_number=$events[$i]{sn}: " + . "got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } + + # Property (c): every cross-source-file boundary event must carry + # last_committed = sequence_number - 1. By protocol the first event + # of each source binlog has source-side last_committed=0; if the + # rewriter's segment-boundary fix is missing, that 0 propagates + # verbatim into the local file. With the fix the value is replaced + # with the immediately preceding local sequence_number, restoring + # the source's "rotation = synchronization point" semantics that + # would otherwise be lost when we coalesce A, B, C into one local + # file. + # + # Source A contributes 10 GTIDs (CREATE TABLE + 9 INSERTs), source B + # contributes 5, source C contributes 3 - boundary events are + # therefore at sequence_number 11 (first of B) and 16 (first of C). + for my $boundary_seq (11, 16) { + my $idx = $boundary_seq - 1; + my $expected_lc = $boundary_seq - 1; + die "boundary-fix violation at sequence_number=$boundary_seq: " + . "expected last_committed=$expected_lc, got " + . "last_committed=$events[$idx]{lc}" + unless $events[$idx]{lc} == $expected_lc; + } +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test new file mode 100644 index 0000000..07d1ec4 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test @@ -0,0 +1,145 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion test to gtid_renumbering.test exercising the OPPOSITE +# side of the rewrite-mode renumberer: a single source binlog whose +# transactions get split across multiple local binlog files because +# the configured rewrite_file_size is much smaller than the total +# stream size. The renumberer must: +# 1. restart sequence_number at 1 in every new local file (otherwise +# sequence_number would keep growing across local files, leaving +# the consumer with a logical clock the local PREVIOUS_GTIDS +# header cannot describe); +# 2. only ever emit last_committed values that reference an event +# already present in the SAME local file (i.e. < sequence_number). +# +# We verify (1) and (2) on the first two local files; observing local +# rotation at all is what makes this test meaningful, so we also +# require that bnlg.000002 actually got created. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +--source ../include/identify_storage_backend.inc + +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +--let $binsrv_checkpoint_size = 1 +# small rewrite_file_size forces frequent local rotations +--let $binsrv_rewrite_file_size = 1K +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Building a single-source-binlog workload that is large enough +--echo *** to span multiple 1K local binlog files after rewrite. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, payload VARCHAR(100) NOT NULL, PRIMARY KEY(id)) ENGINE=InnoDB; + +--disable_query_log +--let $i = 1 +while ($i <= 60) +{ + INSERT INTO t1 (payload) VALUES (REPEAT('x', 80)); + --inc $i +} +--enable_query_log + +--echo +--echo *** Executing the Binlog Server utility in rewrite mode. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Asserting that the rewriter actually rotated locally +--echo *** (bnlg.000002 must exist - otherwise the test does not +--echo *** exercise the local-rotation reset code path). +if ($storage_backend == file) +{ + --file_exists $binsrv_storage_path/bnlg.000002 + --let $first_local_path = $binsrv_storage_path/bnlg.000001 + --let $second_local_path = $binsrv_storage_path/bnlg.000002 +} +if ($storage_backend == s3) +{ + --let $first_local_path = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.bnlg.000001 + --let $second_local_path = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.bnlg.000002 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $first_local_path > /dev/null + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000002 $second_local_path > /dev/null +} + +--echo +--echo *** Validating per-local-file sequence_number reset and +--echo *** last_committed range. Aborts with --die on violation; +--echo *** produces no output otherwise. + +--let FIRST_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.first.dump +--let SECOND_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.second.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $first_local_path > $FIRST_DUMP +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $second_local_path > $SECOND_DUMP + +--perl + use strict; + use warnings; + + sub parse_gtid_dump { + my ($path) = @_; + open(my $fh, '<', $path) or die "Failed to open $path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + return @events; + } + + sub validate_local_file { + my ($label, $events_ref) = @_; + my @events = @$events_ref; + die "$label: no GTID events found in dump" unless @events > 0; + + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + + # sequence_number must restart at 1 and grow gap-free in EACH + # local file + die "$label: sequence_number gap or duplicate at event index $i: " + . "expected $expected_sn, got $events[$i]{sn}" + unless $events[$i]{sn} == $expected_sn; + + # last_committed must always reference an in-file + # sequence_number + die "$label: last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 + && $events[$i]{lc} < $events[$i]{sn}; + } + } + + my @first = parse_gtid_dump($ENV{'FIRST_DUMP'}); + my @second = parse_gtid_dump($ENV{'SECOND_DUMP'}); + + validate_local_file("bnlg.000001", \@first); + validate_local_file("bnlg.000002", \@second); +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $FIRST_DUMP +--remove_file $SECOND_DUMP +if ($storage_backend == s3) +{ + --remove_file $first_local_path + --remove_file $second_local_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume.test b/mtr/binlog_streaming/t/gtid_renumbering_resume.test new file mode 100644 index 0000000..d2a0e8f --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume.test @@ -0,0 +1,205 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion test to gtid_renumbering.test exercising the +# RESUME / RESTART path of the rewrite-mode renumberer. +# +# In rewrite mode the renumberer maintains an in-memory +# next_local_seq counter that allocates sequence_number values inside +# the current local binlog file. When the binsrv utility exits while +# a local file is still open and is later restarted (or when a +# reconnect drops the in-process state), the counter must continue +# forward instead of resetting to 1 - otherwise the post-resume +# events would collide with the sequence_number values already +# emitted into the same local file. +# +# The implementation persists next_local_seq (and last_emitted_offset) +# in the per-binlog .json metadata file at every checkpoint flush. +# On the next binsrv invocation, the storage layer hands that +# snapshot back to the renumberer via resume_in_existing_local_file() +# right after storage construction. +# +# This test verifies the property end-to-end: +# 1. Run binsrv against a partially-populated source, producing N +# GTID events with sequence_number 1..N in bnlg.000001. +# 2. Generate more transactions on the source, with a FLUSH BINARY +# LOGS in between so the second invocation also has to absorb +# a source-side rotation right after resume. +# 3. Run binsrv a second time. Expected: bnlg.000001 now contains +# N+M events with sequence_number 1..N+M (gap-free, no +# duplicates). +# 4. Cross-source-file boundary event at index N must carry +# last_committed = N (not 0): the segment-boundary fix has to +# keep working across a resume. +# +# Without the persisted-renumberer-state recovery, step 3 would +# instead produce sequence_number 1..N followed by 1..M - which the +# perl validator catches as a sequence_number collision. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +# checkpoint_size = 1 forces a metadata flush after every transaction +# boundary, so the renumberer recovery snapshot persisted to the +# metadata file is always in lockstep with the binlog content. With a +# larger checkpoint_size the test would still work for graceful exits +# but would be racy under abrupt termination - we want the strongest +# guarantee here. +--let $binsrv_checkpoint_size = 1 +# rewrite_file_size set high enough that all transactions land in a +# single local file - the point of this test is that the renumberer's +# counter MUST continue forward across binsrv invocations within the +# same local file, not that we exercise local rotation +# (gtid_renumbering_local_rotation.test covers that). +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Phase 1: CREATE TABLE + 4 INSERTs in source binlog A +--echo *** (5 GTID events). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +--disable_query_log +--let $i = 1 +while ($i <= 4) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** First binsrv invocation: writes 5 GTID events with +--echo *** sequence_number 1..5 to bnlg.000001 and persists the +--echo *** renumberer recovery snapshot (next_local_seq=5) in the +--echo *** per-binlog metadata file. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Flushing source binary log between invocations so the +--echo *** second binsrv run also has to absorb a source-side +--echo *** rotation right after resuming. +FLUSH BINARY LOGS; + +--echo +--echo *** Phase 2: 5 INSERTs in source binlog B (5 more GTID events). +--disable_query_log +--let $i = 1 +while ($i <= 5) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Second binsrv invocation: must resume the renumberer from +--echo *** the persisted snapshot (next_local_seq=5) and append +--echo *** sequence_number 6..10 to bnlg.000001. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing the local binlog file produced across the two +--echo *** invocations and dumping it via mysqlbinlog for textual +--echo *** inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering_resume.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_resume.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating that sequence_number is gap-free across the +--echo *** binsrv-invocation boundary and that the post-resume +--echo *** source-binlog-boundary event has last_committed = +--echo *** sequence_number - 1. Aborts with --die on any invariant +--echo *** violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # Phase 1 contributes 5 GTIDs (CREATE TABLE + 4 INSERTs); phase 2 + # contributes 5 INSERTs - 10 events in total, all in bnlg.000001. + my $expected_count = 10; + die "expected exactly $expected_count GTID events in the local " + . "binlog file, got " . scalar @events + unless @events == $expected_count; + + # Property 1: sequence_number must be gap-free across the + # invocation boundary - the renumberer in the second invocation + # must start at 6, not at 1. A regression of the persisted-state + # recovery would surface as duplicate sequence_number values + # (1..5 from each invocation). + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got " + . "$events[$i]{sn} (resume regression: renumberer most " + . "likely restarted at 1 instead of resuming from 6)" + unless $events[$i]{sn} == $expected_sn; + + # Property 2: last_committed must reference an in-file + # sequence_number, i.e. lie in [0, sequence_number - 1]. + die "last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } + + # Property 3: the very first event of source binlog B (the first + # event the SECOND binsrv invocation writes) is the cross-source- + # file boundary; its source-side last_committed is 0 by protocol + # and the renumberer must replace it with sequence_number - 1 to + # preserve the cross-rotation synchronization point. The boundary + # event lies at sequence_number = 6 (first event of phase 2). + my $boundary_seq = 6; + my $idx = $boundary_seq - 1; + my $expected_lc = $boundary_seq - 1; + die "boundary-fix violation at sequence_number=$boundary_seq " + . "(post-resume): expected last_committed=$expected_lc, got " + . "last_committed=$events[$idx]{lc}" + unless $events[$idx]{lc} == $expected_lc; +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test new file mode 100644 index 0000000..621aa6f --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test @@ -0,0 +1,224 @@ +--source ../include/have_binsrv.inc + +--source include/have_debug_sync.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion to gtid_renumbering_resume.test exercising the +# DURABILITY of the persisted renumberer recovery snapshot under a +# mid-transaction disconnect in rewrite mode. +# +# In rewrite mode the renumberer's per-file sequence_number counter +# is bumped speculatively the moment a GTID event is rewritten, but +# it is only "committed" (and mirrored into the per-binlog .json +# metadata) at the corresponding transaction boundary. If a connection +# drops between a GTID rewrite and the matching transaction boundary, +# the partial transaction is discarded and the renumberer must be +# rolled back to the previously committed state - otherwise the +# snapshot persisted alongside the durable bytes would point past the +# end of those bytes, and the next binsrv invocation would re-allocate +# a sequence_number for the resent transaction, leaving a permanent +# gap in the local sequence_number stream. +# +# The scenario: +# 1. binsrv processes T_first (CREATE TABLE t1) cleanly. Disk: +# bnlg.000001 contains the GTID event with sequence_number=1 +# and the metadata records next_local_seq=1. +# 2. The source-side dump thread is armed with +# 'dump_thread_wait_after_send_write_rows'. From now on it +# will pause right after sending the first WRITE_ROWS event +# of any transaction. +# 3. The source produces T_second (CREATE TABLE t2, no +# Write_rows) and T_third (INSERT INTO t1, has Write_rows). +# 4. The second binsrv invocation absorbs T_second in full but +# freezes mid-T_third (after T_third's GTID has already +# advanced the renumberer's speculative counter from 2 to 3). +# It then hits read_timeout and exits with a non-zero code. +# The storage destructor flushes whatever was complete in the +# in-memory buffer - i.e. T_second - and writes the per-binlog +# .json metadata. +# 5. The persisted snapshot at this point MUST carry +# next_local_seq=2 (the committed value, matching T_second +# which is the last GTID actually flushed to disk). With a +# regression of the commit/rollback semantics the snapshot +# would instead carry the speculative next_local_seq=3 - a +# value that points past the durable bytes. +# 6. The third binsrv invocation re-reads the metadata, seeds +# the renumberer accordingly and re-receives T_third (which +# the source resends because binsrv never acknowledged it). +# With the fix T_third lands as sequence_number=3 and +# bnlg.000001 ends up with the contiguous sequence +# 1, 2, 3. Without the fix it lands as sequence_number=4 and +# the perl validator at the end of this test catches the gap. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +# Short read_timeout to make the second binsrv invocation exit +# quickly once the source dump thread freezes mid-transaction. +--let $binsrv_read_timeout = 5 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +# Large checkpoint_size so flushes happen only at end-of-stream +# (storage destructor). With this configuration multiple complete +# transactions can stay buffered alongside an in-flight one - the +# layout that exposes the stale-snapshot bug when the disconnect +# happens mid-transaction. +--let $binsrv_checkpoint_size = 1G +# Single local file for the entire test - the property under test +# is the in-file recovery snapshot, not local rotation. +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Phase 1: T_first = CREATE TABLE t1 (1 GTID event, no +--echo *** Write_rows). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +--echo +--echo *** First binsrv invocation: writes T_first with +--echo *** sequence_number=1 to bnlg.000001 and persists +--echo *** next_local_seq=1 in the per-binlog metadata. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Phase 2: arm the source dump thread to pause AFTER +--echo *** sending the first WRITE_ROWS event of any transaction. +--echo *** Combined with the short binsrv read_timeout, this freezes +--echo *** the second binsrv invocation mid-T_third (after T_third's +--echo *** GTID has been processed by binsrv but before T_third's +--echo *** XID can be received). +SET @old_global_debug = @@global.debug; +SET GLOBAL DEBUG = '+d,dump_thread_wait_after_send_write_rows'; + +--echo +--echo *** Workload that the second binsrv invocation will read: +--echo *** T_second (CREATE TABLE t2, no Write_rows; will be +--echo *** absorbed in full) and T_third (INSERT INTO t1, has +--echo *** Write_rows; binsrv will receive its GTID and prefix +--echo *** events and then time out waiting for the rest). +CREATE TABLE t2(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +INSERT INTO t1 VALUES(DEFAULT); + +--echo +--echo *** Second binsrv invocation: expected to exit with non-zero +--echo *** status after read_timeout. The storage destructor flushes +--echo *** the COMMITTED transaction (T_second) and persists the +--echo *** per-binlog metadata. The renumberer recovery snapshot +--echo *** persisted here MUST be next_local_seq=2 (committed, in +--echo *** lockstep with the flushed bytes) and not the speculative +--echo *** next_local_seq=3 produced by T_third's GTID rewrite. +--error 1 +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Releasing the paused source dump thread: SIGNAL wakes it +--echo *** from the debug_sync wait, after which its first attempt +--echo *** to send the next event fails on the half-closed socket +--echo *** and the dump thread exits. DEBUG flag is then restored so +--echo *** the new dump thread spawned by binsrv 3 does not pause. +SET DEBUG_SYNC = 'now SIGNAL signal.continue'; +SET GLOBAL DEBUG = @old_global_debug; + +let $wait_condition = SELECT COUNT(*) = 0 FROM information_schema.processlist WHERE COMMAND IN ('Binlog Dump', 'Binlog Dump GTID'); +--source include/wait_condition.inc + +SET DEBUG_SYNC = 'RESET'; + +--echo +--echo *** Third binsrv invocation: the source resends T_third +--echo *** (binsrv never acknowledged it), and the renumberer +--echo *** continues from the persisted committed snapshot. T_third +--echo *** must land as sequence_number=3 so bnlg.000001 ends up +--echo *** gap-free. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing bnlg.000001 produced across all three +--echo *** invocations and dumping it via mysqlbinlog for textual +--echo *** inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering_resume_after_partial.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_resume_after_partial.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating that bnlg.000001 contains exactly 3 GTID +--echo *** events with contiguous sequence_numbers 1, 2, 3 and +--echo *** that every last_committed value references an in-file +--echo *** sequence_number. Aborts with --die on any invariant +--echo *** violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # T_first (CREATE TABLE t1), T_second (CREATE TABLE t2), and the + # resent T_third (INSERT INTO t1) - 3 GTID events in total, all + # in bnlg.000001. + my $expected_count = 3; + die "expected exactly $expected_count GTID events in the local " + . "binlog file, got " . scalar @events + unless @events == $expected_count; + + # Property: sequence_number must be gap-free across the + # mid-transaction disconnect. A regression of the commit/rollback + # of the renumberer's recovery snapshot would surface as + # sequence_number=4 (instead of 3) at index 2 - the persisted + # snapshot would have advanced past the discarded T_third and + # caused the resent T_third to be allocated next_local_seq+1 + # twice. + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got " + . "$events[$i]{sn} (regression: persisted recovery info " + . "advanced past the discarded transaction's bytes)" + unless $events[$i]{sn} == $expected_sn; + + die "last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the tables. +DROP TABLE t1, t2; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc