diff --git a/mtr/binlog_streaming/r/gtid_renumbering.result b/mtr/binlog_streaming/r/gtid_renumbering.result new file mode 100644 index 0000000..4fb6649 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering.result @@ -0,0 +1,47 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Building deterministic binlog content with two source-binlog +*** rotations between transaction groups. + +*** Source binlog A: CREATE TABLE + 9 INSERTs (10 GTIDs total). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** Flushing source binary log (introduces the first +*** source-binlog rotation that the rewriter must absorb). +FLUSH BINARY LOGS; + +*** Source binlog B: 5 INSERTs. + +*** Flushing source binary log one more time (second +*** source-binlog rotation absorbed inside the same local file). +FLUSH BINARY LOGS; + +*** Source binlog C: 3 INSERTs. + +*** Executing the Binlog Server utility in rewrite mode. + +*** Materializing the local binlog file produced by the rewriter +*** and dumping it via mysqlbinlog for textual inspection. + +*** Validating (sequence_number, last_committed) of every GTID +*** event in the local binlog file. Aborts with --die on any +*** invariant violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result b/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result new file mode 100644 index 0000000..abd2418 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_local_rotation.result @@ -0,0 +1,34 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Building a single-source-binlog workload that is large enough +*** to span multiple 1K local binlog files after rewrite. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, payload VARCHAR(100) NOT NULL, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** Executing the Binlog Server utility in rewrite mode. + +*** Asserting that the rewriter actually rotated locally +*** (bnlg.000002 must exist - otherwise the test does not +*** exercise the local-rotation reset code path). + +*** Validating per-local-file sequence_number reset and +*** last_committed range. Aborts with --die on violation; +*** produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_resume.result b/mtr/binlog_streaming/r/gtid_renumbering_resume.result new file mode 100644 index 0000000..107c847 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_resume.result @@ -0,0 +1,50 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Phase 1: CREATE TABLE + 4 INSERTs in source binlog A +*** (5 GTID events). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** First binsrv invocation: writes 5 GTID events with +*** sequence_number 1..5 to bnlg.000001 and persists the +*** renumberer recovery snapshot (next_local_seq=5) in the +*** per-binlog metadata file. + +*** Flushing source binary log between invocations so the +*** second binsrv run also has to absorb a source-side +*** rotation right after resuming. +FLUSH BINARY LOGS; + +*** Phase 2: 5 INSERTs in source binlog B (5 more GTID events). + +*** Second binsrv invocation: must resume the renumberer from +*** the persisted snapshot (next_local_seq=5) and append +*** sequence_number 6..10 to bnlg.000001. + +*** Materializing the local binlog file produced across the two +*** invocations and dumping it via mysqlbinlog for textual +*** inspection. + +*** Validating that sequence_number is gap-free across the +*** binsrv-invocation boundary and that the post-resume +*** source-binlog-boundary event has last_committed = +*** sequence_number - 1. Aborts with --die on any invariant +*** violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the table. +DROP TABLE t1; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result b/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result new file mode 100644 index 0000000..d868470 --- /dev/null +++ b/mtr/binlog_streaming/r/gtid_renumbering_resume_after_partial.result @@ -0,0 +1,78 @@ +*** Resetting replication at the very beginning of the test. + +*** Generating a configuration file in JSON format for the Binlog +*** Server utility. + +*** Determining binlog file directory from the server. + +*** Creating a temporary directory for storing +*** binlog files downloaded via the Binlog Server utility. + +*** Phase 1: T_first = CREATE TABLE t1 (1 GTID event, no +*** Write_rows). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +*** First binsrv invocation: writes T_first with +*** sequence_number=1 to bnlg.000001 and persists +*** next_local_seq=1 in the per-binlog metadata. + +*** Phase 2: arm the source dump thread to pause AFTER +*** sending the first WRITE_ROWS event of any transaction. +*** Combined with the short binsrv read_timeout, this freezes +*** the second binsrv invocation mid-T_third (after T_third's +*** GTID has been processed by binsrv but before T_third's +*** XID can be received). +SET @old_global_debug = @@global.debug; +SET GLOBAL DEBUG = '+d,dump_thread_wait_after_send_write_rows'; + +*** Workload that the second binsrv invocation will read: +*** T_second (CREATE TABLE t2, no Write_rows; will be +*** absorbed in full) and T_third (INSERT INTO t1, has +*** Write_rows; binsrv will receive its GTID and prefix +*** events and then time out waiting for the rest). +CREATE TABLE t2(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +INSERT INTO t1 VALUES(DEFAULT); + +*** Second binsrv invocation: expected to exit with non-zero +*** status after read_timeout. The storage destructor flushes +*** the COMMITTED transaction (T_second) and persists the +*** per-binlog metadata. The renumberer recovery snapshot +*** persisted here MUST be next_local_seq=2 (committed, in +*** lockstep with the flushed bytes) and not the speculative +*** next_local_seq=3 produced by T_third's GTID rewrite. + +*** Releasing the paused source dump thread: SIGNAL wakes it +*** from the debug_sync wait, after which its first attempt +*** to send the next event fails on the half-closed socket +*** and the dump thread exits. DEBUG flag is then restored so +*** the new dump thread spawned by binsrv 3 does not pause. +SET DEBUG_SYNC = 'now SIGNAL signal.continue'; +SET GLOBAL DEBUG = @old_global_debug; +SET DEBUG_SYNC = 'RESET'; + +*** Third binsrv invocation: the source resends T_third +*** (binsrv never acknowledged it), and the renumberer +*** continues from the persisted committed snapshot. T_third +*** must land as sequence_number=3 so bnlg.000001 ends up +*** gap-free. + +*** Materializing bnlg.000001 produced across all three +*** invocations and dumping it via mysqlbinlog for textual +*** inspection. + +*** Validating that bnlg.000001 contains exactly 3 GTID +*** events with contiguous sequence_numbers 1, 2, 3 and +*** that every last_committed value references an in-file +*** sequence_number. Aborts with --die on any invariant +*** violation; produces no output otherwise. + +*** Removing temporary files. + +*** Dropping the tables. +DROP TABLE t1, t2; + +*** Removing the Binlog Server utility storage directory. + +*** Removing the Binlog Server utility log file. + +*** Removing the Binlog Server utility configuration file. diff --git a/mtr/binlog_streaming/t/gtid_renumbering-master.opt b/mtr/binlog_streaming/t/gtid_renumbering-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering.test b/mtr/binlog_streaming/t/gtid_renumbering.test new file mode 100644 index 0000000..12b8836 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering.test @@ -0,0 +1,209 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# This test exercises the rewrite-mode GTID renumberer. In rewrite mode +# the binlog server discards the source's ROTATE / FORMAT_DESCRIPTION / +# PREVIOUS_GTIDS / STOP events and coalesces transactions into local +# binlog files of a configurable size. Two consequences: +# 1. Multiple source binlogs may be folded into a single local file. +# The source resets sequence_number to 1 at every rotation, which +# would produce duplicate / non-monotone sequence_number values in +# our local file unless the rewriter renumbers them. +# 2. The first event of each new source-file segment carries +# last_committed=0 from the source, relying on the implicit +# synchronization guarantee that every transaction of the previous +# source file commits before any transaction of the next file +# starts. After coalescing, that physical boundary is gone, so the +# rewriter must replace last_committed=0 with sequence_number-1 to +# preserve the cross-file commit ordering on the consumer side. +# +# We craft three source binlogs (10 + 5 + 3 GTIDs) coalesced into a +# single local file (1G rewrite_file_size) and verify that: +# (a) sequence_number starts at 1 and advances by exactly 1 per event +# (renumbering covers all source-side rotations the rewriter +# absorbed - without the renumberer we would see duplicate +# sequence_number=1 values from each source file); +# (b) last_committed is in [0, sequence_number - 1] for every event; +# (c) the cross-source-file boundary events (events #11 and #16 - +# the first transactions from source binlogs B and C) carry +# a non-zero last_committed (specifically, sequence_number - 1). +# Without the rewriter's segment-boundary fix these would inherit +# the source's verbatim last_committed=0 and let the local +# applier reorder them ahead of the previous source file's +# commits. This assertion is robust regardless of the source's +# binlog_transaction_dependency_tracking mode. + +# in case of --repeat=N, we need to start from a fresh binary log to make +# this test deterministic +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +--let $binsrv_checkpoint_size = 1 +# rewrite_file_size set high enough that all transactions land in a +# single local file - the point of this test is to exercise multiple +# source-file boundaries WITHIN the same local file +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Building deterministic binlog content with two source-binlog +--echo *** rotations between transaction groups. +--echo +--echo *** Source binlog A: CREATE TABLE + 9 INSERTs (10 GTIDs total). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +--disable_query_log +--let $i = 1 +while ($i <= 9) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Flushing source binary log (introduces the first +--echo *** source-binlog rotation that the rewriter must absorb). +FLUSH BINARY LOGS; + +--echo +--echo *** Source binlog B: 5 INSERTs. +--disable_query_log +--let $i = 1 +while ($i <= 5) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Flushing source binary log one more time (second +--echo *** source-binlog rotation absorbed inside the same local file). +FLUSH BINARY LOGS; + +--echo +--echo *** Source binlog C: 3 INSERTs. +--disable_query_log +--let $i = 1 +while ($i <= 3) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Executing the Binlog Server utility in rewrite mode. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing the local binlog file produced by the rewriter +--echo *** and dumping it via mysqlbinlog for textual inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating (sequence_number, last_committed) of every GTID +--echo *** event in the local binlog file. Aborts with --die on any +--echo *** invariant violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # Source A contributes 10 GTIDs (CREATE TABLE + 9 INSERTs), source B + # contributes 5, source C contributes 3 - 18 events in total. + my $expected_count = 18; + die "expected exactly $expected_count GTID events in the local binlog " + . "file, got " . scalar @events + unless @events == $expected_count; + + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + + # Property (a): per-local-file sequence_number must restart at 1 + # and advance gap-free by exactly 1 per event - this is what + # protects us from the source restarting its sequence_number at + # every rotation that we absorb. + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got $events[$i]{sn}" + unless $events[$i]{sn} == $expected_sn; + + # Property (b): last_committed must reference an in-file + # sequence_number, i.e. lie in [0, sequence_number - 1]. A value + # >= sequence_number would point to the future; a negative value + # is impossible on the wire and would indicate a corrupt rewrite. + die "last_committed out of range at sequence_number=$events[$i]{sn}: " + . "got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } + + # Property (c): every cross-source-file boundary event must carry + # last_committed = sequence_number - 1. By protocol the first event + # of each source binlog has source-side last_committed=0; if the + # rewriter's segment-boundary fix is missing, that 0 propagates + # verbatim into the local file. With the fix the value is replaced + # with the immediately preceding local sequence_number, restoring + # the source's "rotation = synchronization point" semantics that + # would otherwise be lost when we coalesce A, B, C into one local + # file. + # + # Source A contributes 10 GTIDs (CREATE TABLE + 9 INSERTs), source B + # contributes 5, source C contributes 3 - boundary events are + # therefore at sequence_number 11 (first of B) and 16 (first of C). + for my $boundary_seq (11, 16) { + my $idx = $boundary_seq - 1; + my $expected_lc = $boundary_seq - 1; + die "boundary-fix violation at sequence_number=$boundary_seq: " + . "expected last_committed=$expected_lc, got " + . "last_committed=$events[$idx]{lc}" + unless $events[$idx]{lc} == $expected_lc; + } +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test new file mode 100644 index 0000000..07d1ec4 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_local_rotation.test @@ -0,0 +1,145 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion test to gtid_renumbering.test exercising the OPPOSITE +# side of the rewrite-mode renumberer: a single source binlog whose +# transactions get split across multiple local binlog files because +# the configured rewrite_file_size is much smaller than the total +# stream size. The renumberer must: +# 1. restart sequence_number at 1 in every new local file (otherwise +# sequence_number would keep growing across local files, leaving +# the consumer with a logical clock the local PREVIOUS_GTIDS +# header cannot describe); +# 2. only ever emit last_committed values that reference an event +# already present in the SAME local file (i.e. < sequence_number). +# +# We verify (1) and (2) on the first two local files; observing local +# rotation at all is what makes this test meaningful, so we also +# require that bnlg.000002 actually got created. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +--source ../include/identify_storage_backend.inc + +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +--let $binsrv_checkpoint_size = 1 +# small rewrite_file_size forces frequent local rotations +--let $binsrv_rewrite_file_size = 1K +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Building a single-source-binlog workload that is large enough +--echo *** to span multiple 1K local binlog files after rewrite. +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, payload VARCHAR(100) NOT NULL, PRIMARY KEY(id)) ENGINE=InnoDB; + +--disable_query_log +--let $i = 1 +while ($i <= 60) +{ + INSERT INTO t1 (payload) VALUES (REPEAT('x', 80)); + --inc $i +} +--enable_query_log + +--echo +--echo *** Executing the Binlog Server utility in rewrite mode. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Asserting that the rewriter actually rotated locally +--echo *** (bnlg.000002 must exist - otherwise the test does not +--echo *** exercise the local-rotation reset code path). +if ($storage_backend == file) +{ + --file_exists $binsrv_storage_path/bnlg.000002 + --let $first_local_path = $binsrv_storage_path/bnlg.000001 + --let $second_local_path = $binsrv_storage_path/bnlg.000002 +} +if ($storage_backend == s3) +{ + --let $first_local_path = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.bnlg.000001 + --let $second_local_path = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.bnlg.000002 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $first_local_path > /dev/null + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000002 $second_local_path > /dev/null +} + +--echo +--echo *** Validating per-local-file sequence_number reset and +--echo *** last_committed range. Aborts with --die on violation; +--echo *** produces no output otherwise. + +--let FIRST_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.first.dump +--let SECOND_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_local_rotation.second.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $first_local_path > $FIRST_DUMP +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $second_local_path > $SECOND_DUMP + +--perl + use strict; + use warnings; + + sub parse_gtid_dump { + my ($path) = @_; + open(my $fh, '<', $path) or die "Failed to open $path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + return @events; + } + + sub validate_local_file { + my ($label, $events_ref) = @_; + my @events = @$events_ref; + die "$label: no GTID events found in dump" unless @events > 0; + + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + + # sequence_number must restart at 1 and grow gap-free in EACH + # local file + die "$label: sequence_number gap or duplicate at event index $i: " + . "expected $expected_sn, got $events[$i]{sn}" + unless $events[$i]{sn} == $expected_sn; + + # last_committed must always reference an in-file + # sequence_number + die "$label: last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 + && $events[$i]{lc} < $events[$i]{sn}; + } + } + + my @first = parse_gtid_dump($ENV{'FIRST_DUMP'}); + my @second = parse_gtid_dump($ENV{'SECOND_DUMP'}); + + validate_local_file("bnlg.000001", \@first); + validate_local_file("bnlg.000002", \@second); +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $FIRST_DUMP +--remove_file $SECOND_DUMP +if ($storage_backend == s3) +{ + --remove_file $first_local_path + --remove_file $second_local_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume.test b/mtr/binlog_streaming/t/gtid_renumbering_resume.test new file mode 100644 index 0000000..d2a0e8f --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume.test @@ -0,0 +1,205 @@ +--source ../include/have_binsrv.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion test to gtid_renumbering.test exercising the +# RESUME / RESTART path of the rewrite-mode renumberer. +# +# In rewrite mode the renumberer maintains an in-memory +# next_local_seq counter that allocates sequence_number values inside +# the current local binlog file. When the binsrv utility exits while +# a local file is still open and is later restarted (or when a +# reconnect drops the in-process state), the counter must continue +# forward instead of resetting to 1 - otherwise the post-resume +# events would collide with the sequence_number values already +# emitted into the same local file. +# +# The implementation persists next_local_seq (and last_emitted_offset) +# in the per-binlog .json metadata file at every checkpoint flush. +# On the next binsrv invocation, the storage layer hands that +# snapshot back to the renumberer via resume_in_existing_local_file() +# right after storage construction. +# +# This test verifies the property end-to-end: +# 1. Run binsrv against a partially-populated source, producing N +# GTID events with sequence_number 1..N in bnlg.000001. +# 2. Generate more transactions on the source, with a FLUSH BINARY +# LOGS in between so the second invocation also has to absorb +# a source-side rotation right after resume. +# 3. Run binsrv a second time. Expected: bnlg.000001 now contains +# N+M events with sequence_number 1..N+M (gap-free, no +# duplicates). +# 4. Cross-source-file boundary event at index N must carry +# last_committed = N (not 0): the segment-boundary fix has to +# keep working across a resume. +# +# Without the persisted-renumberer-state recovery, step 3 would +# instead produce sequence_number 1..N followed by 1..M - which the +# perl validator catches as a sequence_number collision. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +--let $binsrv_read_timeout = 60 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +# checkpoint_size = 1 forces a metadata flush after every transaction +# boundary, so the renumberer recovery snapshot persisted to the +# metadata file is always in lockstep with the binlog content. With a +# larger checkpoint_size the test would still work for graceful exits +# but would be racy under abrupt termination - we want the strongest +# guarantee here. +--let $binsrv_checkpoint_size = 1 +# rewrite_file_size set high enough that all transactions land in a +# single local file - the point of this test is that the renumberer's +# counter MUST continue forward across binsrv invocations within the +# same local file, not that we exercise local rotation +# (gtid_renumbering_local_rotation.test covers that). +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Phase 1: CREATE TABLE + 4 INSERTs in source binlog A +--echo *** (5 GTID events). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +--disable_query_log +--let $i = 1 +while ($i <= 4) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** First binsrv invocation: writes 5 GTID events with +--echo *** sequence_number 1..5 to bnlg.000001 and persists the +--echo *** renumberer recovery snapshot (next_local_seq=5) in the +--echo *** per-binlog metadata file. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Flushing source binary log between invocations so the +--echo *** second binsrv run also has to absorb a source-side +--echo *** rotation right after resuming. +FLUSH BINARY LOGS; + +--echo +--echo *** Phase 2: 5 INSERTs in source binlog B (5 more GTID events). +--disable_query_log +--let $i = 1 +while ($i <= 5) +{ + INSERT INTO t1 VALUES(DEFAULT); + --inc $i +} +--enable_query_log + +--echo +--echo *** Second binsrv invocation: must resume the renumberer from +--echo *** the persisted snapshot (next_local_seq=5) and append +--echo *** sequence_number 6..10 to bnlg.000001. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing the local binlog file produced across the two +--echo *** invocations and dumping it via mysqlbinlog for textual +--echo *** inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering_resume.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_resume.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating that sequence_number is gap-free across the +--echo *** binsrv-invocation boundary and that the post-resume +--echo *** source-binlog-boundary event has last_committed = +--echo *** sequence_number - 1. Aborts with --die on any invariant +--echo *** violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # Phase 1 contributes 5 GTIDs (CREATE TABLE + 4 INSERTs); phase 2 + # contributes 5 INSERTs - 10 events in total, all in bnlg.000001. + my $expected_count = 10; + die "expected exactly $expected_count GTID events in the local " + . "binlog file, got " . scalar @events + unless @events == $expected_count; + + # Property 1: sequence_number must be gap-free across the + # invocation boundary - the renumberer in the second invocation + # must start at 6, not at 1. A regression of the persisted-state + # recovery would surface as duplicate sequence_number values + # (1..5 from each invocation). + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got " + . "$events[$i]{sn} (resume regression: renumberer most " + . "likely restarted at 1 instead of resuming from 6)" + unless $events[$i]{sn} == $expected_sn; + + # Property 2: last_committed must reference an in-file + # sequence_number, i.e. lie in [0, sequence_number - 1]. + die "last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } + + # Property 3: the very first event of source binlog B (the first + # event the SECOND binsrv invocation writes) is the cross-source- + # file boundary; its source-side last_committed is 0 by protocol + # and the renumberer must replace it with sequence_number - 1 to + # preserve the cross-rotation synchronization point. The boundary + # event lies at sequence_number = 6 (first event of phase 2). + my $boundary_seq = 6; + my $idx = $boundary_seq - 1; + my $expected_lc = $boundary_seq - 1; + die "boundary-fix violation at sequence_number=$boundary_seq " + . "(post-resume): expected last_committed=$expected_lc, got " + . "last_committed=$events[$idx]{lc}" + unless $events[$idx]{lc} == $expected_lc; +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the table. +DROP TABLE t1; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt new file mode 100644 index 0000000..c7529f0 --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial-master.opt @@ -0,0 +1,2 @@ +--gtid-mode=on +--enforce-gtid-consistency diff --git a/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test new file mode 100644 index 0000000..621aa6f --- /dev/null +++ b/mtr/binlog_streaming/t/gtid_renumbering_resume_after_partial.test @@ -0,0 +1,224 @@ +--source ../include/have_binsrv.inc + +--source include/have_debug_sync.inc + +--source ../include/v80_v84_compatibility_defines.inc + +# Companion to gtid_renumbering_resume.test exercising the +# DURABILITY of the persisted renumberer recovery snapshot under a +# mid-transaction disconnect in rewrite mode. +# +# In rewrite mode the renumberer's per-file sequence_number counter +# is bumped speculatively the moment a GTID event is rewritten, but +# it is only "committed" (and mirrored into the per-binlog .json +# metadata) at the corresponding transaction boundary. If a connection +# drops between a GTID rewrite and the matching transaction boundary, +# the partial transaction is discarded and the renumberer must be +# rolled back to the previously committed state - otherwise the +# snapshot persisted alongside the durable bytes would point past the +# end of those bytes, and the next binsrv invocation would re-allocate +# a sequence_number for the resent transaction, leaving a permanent +# gap in the local sequence_number stream. +# +# The scenario: +# 1. binsrv processes T_first (CREATE TABLE t1) cleanly. Disk: +# bnlg.000001 contains the GTID event with sequence_number=1 +# and the metadata records next_local_seq=1. +# 2. The source-side dump thread is armed with +# 'dump_thread_wait_after_send_write_rows'. From now on it +# will pause right after sending the first WRITE_ROWS event +# of any transaction. +# 3. The source produces T_second (CREATE TABLE t2, no +# Write_rows) and T_third (INSERT INTO t1, has Write_rows). +# 4. The second binsrv invocation absorbs T_second in full but +# freezes mid-T_third (after T_third's GTID has already +# advanced the renumberer's speculative counter from 2 to 3). +# It then hits read_timeout and exits with a non-zero code. +# The storage destructor flushes whatever was complete in the +# in-memory buffer - i.e. T_second - and writes the per-binlog +# .json metadata. +# 5. The persisted snapshot at this point MUST carry +# next_local_seq=2 (the committed value, matching T_second +# which is the last GTID actually flushed to disk). With a +# regression of the commit/rollback semantics the snapshot +# would instead carry the speculative next_local_seq=3 - a +# value that points past the durable bytes. +# 6. The third binsrv invocation re-reads the metadata, seeds +# the renumberer accordingly and re-receives T_third (which +# the source resends because binsrv never acknowledged it). +# With the fix T_third lands as sequence_number=3 and +# bnlg.000001 ends up with the contiguous sequence +# 1, 2, 3. Without the fix it lands as sequence_number=4 and +# the perl validator at the end of this test catches the gap. + +--echo *** Resetting replication at the very beginning of the test. +--disable_query_log +eval $stmt_reset_binary_logs_and_gtids; +--enable_query_log + +# identifying backend storage type ('file' or 's3') +--source ../include/identify_storage_backend.inc + +# creating data directory, configuration file, etc. +--let $binsrv_connect_timeout = 20 +# Short read_timeout to make the second binsrv invocation exit +# quickly once the source dump thread freezes mid-transaction. +--let $binsrv_read_timeout = 5 +--let $binsrv_idle_time = 10 +--let $binsrv_verify_checksum = TRUE +--let $binsrv_replication_mode = gtid +# Large checkpoint_size so flushes happen only at end-of-stream +# (storage destructor). With this configuration multiple complete +# transactions can stay buffered alongside an in-flight one - the +# layout that exposes the stale-snapshot bug when the disconnect +# happens mid-transaction. +--let $binsrv_checkpoint_size = 1G +# Single local file for the entire test - the property under test +# is the in-file recovery snapshot, not local rotation. +--let $binsrv_rewrite_file_size = 1G +--source ../include/set_up_binsrv_environment.inc + +--echo +--echo *** Phase 1: T_first = CREATE TABLE t1 (1 GTID event, no +--echo *** Write_rows). +CREATE TABLE t1(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; + +--echo +--echo *** First binsrv invocation: writes T_first with +--echo *** sequence_number=1 to bnlg.000001 and persists +--echo *** next_local_seq=1 in the per-binlog metadata. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Phase 2: arm the source dump thread to pause AFTER +--echo *** sending the first WRITE_ROWS event of any transaction. +--echo *** Combined with the short binsrv read_timeout, this freezes +--echo *** the second binsrv invocation mid-T_third (after T_third's +--echo *** GTID has been processed by binsrv but before T_third's +--echo *** XID can be received). +SET @old_global_debug = @@global.debug; +SET GLOBAL DEBUG = '+d,dump_thread_wait_after_send_write_rows'; + +--echo +--echo *** Workload that the second binsrv invocation will read: +--echo *** T_second (CREATE TABLE t2, no Write_rows; will be +--echo *** absorbed in full) and T_third (INSERT INTO t1, has +--echo *** Write_rows; binsrv will receive its GTID and prefix +--echo *** events and then time out waiting for the rest). +CREATE TABLE t2(id INT UNSIGNED NOT NULL AUTO_INCREMENT, PRIMARY KEY(id)) ENGINE=InnoDB; +INSERT INTO t1 VALUES(DEFAULT); + +--echo +--echo *** Second binsrv invocation: expected to exit with non-zero +--echo *** status after read_timeout. The storage destructor flushes +--echo *** the COMMITTED transaction (T_second) and persists the +--echo *** per-binlog metadata. The renumberer recovery snapshot +--echo *** persisted here MUST be next_local_seq=2 (committed, in +--echo *** lockstep with the flushed bytes) and not the speculative +--echo *** next_local_seq=3 produced by T_third's GTID rewrite. +--error 1 +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Releasing the paused source dump thread: SIGNAL wakes it +--echo *** from the debug_sync wait, after which its first attempt +--echo *** to send the next event fails on the half-closed socket +--echo *** and the dump thread exits. DEBUG flag is then restored so +--echo *** the new dump thread spawned by binsrv 3 does not pause. +SET DEBUG_SYNC = 'now SIGNAL signal.continue'; +SET GLOBAL DEBUG = @old_global_debug; + +let $wait_condition = SELECT COUNT(*) = 0 FROM information_schema.processlist WHERE COMMAND IN ('Binlog Dump', 'Binlog Dump GTID'); +--source include/wait_condition.inc + +SET DEBUG_SYNC = 'RESET'; + +--echo +--echo *** Third binsrv invocation: the source resends T_third +--echo *** (binsrv never acknowledged it), and the renumberer +--echo *** continues from the persisted committed snapshot. T_third +--echo *** must land as sequence_number=3 so bnlg.000001 ends up +--echo *** gap-free. +--exec $BINSRV fetch $binsrv_config_file_path > /dev/null + +--echo +--echo *** Materializing bnlg.000001 produced across all three +--echo *** invocations and dumping it via mysqlbinlog for textual +--echo *** inspection. +if ($storage_backend == file) +{ + --let $local_binlog_path = $binsrv_storage_path/bnlg.000001 +} +if ($storage_backend == s3) +{ + --let $local_binlog_path = $MYSQL_TMP_DIR/gtid_renumbering_resume_after_partial.bnlg.000001 + --exec $aws_cli s3 cp s3://$aws_s3_bucket$binsrv_storage_path/bnlg.000001 $local_binlog_path > /dev/null +} + +--let GTID_DUMP = $MYSQL_TMP_DIR/gtid_renumbering_resume_after_partial.dump +--exec $MYSQL_BINLOG --base64-output=DECODE-ROWS $local_binlog_path > $GTID_DUMP + +--echo +--echo *** Validating that bnlg.000001 contains exactly 3 GTID +--echo *** events with contiguous sequence_numbers 1, 2, 3 and +--echo *** that every last_committed value references an in-file +--echo *** sequence_number. Aborts with --die on any invariant +--echo *** violation; produces no output otherwise. + +--perl + use strict; + use warnings; + + my $dump_path = $ENV{'GTID_DUMP'}; + open(my $fh, '<', $dump_path) or die "Failed to open $dump_path: $!"; + my @events; + while (my $line = <$fh>) { + if ($line =~ /last_committed=(\d+).*?sequence_number=(\d+)/) { + push @events, { lc => $1 + 0, sn => $2 + 0 }; + } + } + close($fh); + + # T_first (CREATE TABLE t1), T_second (CREATE TABLE t2), and the + # resent T_third (INSERT INTO t1) - 3 GTID events in total, all + # in bnlg.000001. + my $expected_count = 3; + die "expected exactly $expected_count GTID events in the local " + . "binlog file, got " . scalar @events + unless @events == $expected_count; + + # Property: sequence_number must be gap-free across the + # mid-transaction disconnect. A regression of the commit/rollback + # of the renumberer's recovery snapshot would surface as + # sequence_number=4 (instead of 3) at index 2 - the persisted + # snapshot would have advanced past the discarded T_third and + # caused the resent T_third to be allocated next_local_seq+1 + # twice. + for my $i (0 .. $#events) { + my $expected_sn = $i + 1; + die "sequence_number gap or duplicate at event index $i: " + . "expected sequence_number=$expected_sn, got " + . "$events[$i]{sn} (regression: persisted recovery info " + . "advanced past the discarded transaction's bytes)" + unless $events[$i]{sn} == $expected_sn; + + die "last_committed out of range at sequence_number=" + . "$events[$i]{sn}: got last_committed=$events[$i]{lc}" + unless $events[$i]{lc} >= 0 && $events[$i]{lc} < $events[$i]{sn}; + } +EOF + +--echo +--echo *** Removing temporary files. +--remove_file $GTID_DUMP +if ($storage_backend == s3) +{ + --remove_file $local_binlog_path +} + +--echo +--echo *** Dropping the tables. +DROP TABLE t1, t2; + +# cleaning up +--source ../include/tear_down_binsrv_environment.inc