public inbox for [email protected]help / color / mirror / Atom feed
[PATCH] Fix timeline-tracking failure while sending a historic timeline 4+ messages / 3 participants [nested] [flat]
* [PATCH] Fix timeline-tracking failure while sending a historic timeline @ 2021-01-05 04:34 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Kyotaro Horiguchi @ 2021-01-05 04:34 UTC (permalink / raw) Walsender should track timeline switches while sending a historic timeline. Regain that behavior, which was broken in PG13, by a thinko of 709d003fbd. Backpatch to PG13. --- src/backend/replication/walsender.c | 2 +- src/test/perl/PostgresNode.pm | 33 ++++++++++++++++++ src/test/perl/TestLib.pm | 16 ++++++--- src/test/recovery/t/001_stream_rep.pl | 41 ++++++++++++++++++++++- src/test/recovery/t/019_replslot_limit.pl | 37 ++++---------------- 5 files changed, 92 insertions(+), 37 deletions(-) diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 7f87eb7f19..04f6c3ebb4 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2478,7 +2478,7 @@ WalSndSegmentOpen(XLogReaderState *state, XLogSegNo nextSegNo, XLogSegNo endSegNo; XLByteToSeg(sendTimeLineValidUpto, endSegNo, state->segcxt.ws_segsize); - if (state->seg.ws_segno == endSegNo) + if (nextSegNo == endSegNo) *tli_p = sendTimeLineNextTLI; } diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 980f1f1533..687aa3ac88 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -2138,6 +2138,39 @@ sub pg_recvlogical_upto =pod +=item $node->current_log_position() + +Return the current position of server log. + +=cut + +sub current_log_position +{ + my $self = shift; + + return (stat $self->logfile)[7]; +} + +=pod + +=item $node->find_in_log($pattern, $startpos) + +Returns whether the $pattern occurs after $startpos in the server log. + +=cut + +sub find_in_log +{ + my ($self, $pattern, $startpos) = @_; + + $startpos = 0 unless defined $startpos; + my $log = TestLib::slurp_file($self->logfile, $startpos); + + return $log =~ m/$pattern/; +} + +=pod + =back =cut diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pm index a7490d2ce7..a0ce9521e2 100644 --- a/src/test/perl/TestLib.pm +++ b/src/test/perl/TestLib.pm @@ -402,30 +402,38 @@ sub slurp_dir =pod -=item slurp_file(filename) +=item slurp_file(filename, pos) -Return the full contents of the specified file. +Return the contents after pos of the specified file. +Reutrns the full contents if pos is omitted. =cut sub slurp_file { - my ($filename) = @_; + my ($filename, $from) = @_; local $/; my $contents; + + $from = 0 unless defined $from; + if ($Config{osname} ne 'MSWin32') { open(my $in, '<', $filename) or die "could not read \"$filename\": $!"; + seek($in, $from, 0) + or die "could not seek \"$filename\" to $from: $!"; $contents = <$in>; close $in; } else { my $fHandle = createFile($filename, "r", "rwd") - or die "could not open \"$filename\": $^E"; + or die "could not open \"$filename\": $^E\n"; OsFHandleOpen(my $fh = IO::Handle->new(), $fHandle, 'r') or die "could not read \"$filename\": $^E\n"; + seek($fh, $from, 0) + or die "could not seek \"$filename\" to $from: $^E\n"; $contents = <$fh>; CloseHandle($fHandle) or die "could not close \"$filename\": $^E\n"; diff --git a/src/test/recovery/t/001_stream_rep.pl b/src/test/recovery/t/001_stream_rep.pl index 778f11b28b..8d2b24fe55 100644 --- a/src/test/recovery/t/001_stream_rep.pl +++ b/src/test/recovery/t/001_stream_rep.pl @@ -2,8 +2,9 @@ use strict; use warnings; use PostgresNode; +use Time::HiRes qw(usleep); use TestLib; -use Test::More tests => 36; +use Test::More tests => 37; # Initialize master node my $node_master = get_new_node('master'); @@ -409,3 +410,41 @@ ok( ($phys_restart_lsn_pre cmp $phys_restart_lsn_post) == 0, my $master_data = $node_master->data_dir; ok(!-f "$master_data/pg_wal/$segment_removed", "WAL segment $segment_removed recycled after physical slot advancing"); + +# +# Check if timeline-increment works while reading a historic timeline. +my $node_primary_2 = get_new_node('primary_2'); +# archiving is needed to create .paritial segment +$node_primary_2->init(allows_streaming => 1, has_archiving => 1); +$node_primary_2->start; +$node_primary_2->backup($backup_name); +my $node_standby_3 = get_new_node('standby_3'); +$node_standby_3->init_from_backup($node_primary_2, $backup_name, + has_streaming => 1); +$node_primary_2->stop; +$node_primary_2->set_standby_mode; # increment primary timeline +$node_primary_2->start; +$node_primary_2->promote; +my $logstart = $node_standby_3->current_log_position(); +$node_standby_3->start; + +my $success = 0; +for (my $i = 0 ; $i < 1000; $i++) +{ + if ($node_standby_3->find_in_log( + "requested WAL segment [0-9A-F]+ has already been removed", + $logstart)) + { + last; + } + elsif ($node_standby_3->find_in_log( + "End of WAL reached on timeline", + $logstart)) + { + $success = 1; + last; + } + usleep(100_000); +} + +ok($success, 'Timeline increment while reading a historic timeline'); diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl index a7231dcd47..8b3c5de057 100644 --- a/src/test/recovery/t/019_replslot_limit.pl +++ b/src/test/recovery/t/019_replslot_limit.pl @@ -165,19 +165,17 @@ $node_master->wait_for_catchup($node_standby, 'replay', $start_lsn); $node_standby->stop; -ok( !find_in_log( - $node_standby, - "requested WAL segment [0-9A-F]+ has already been removed"), +ok( !$node_standby->find_in_log( + "requested WAL segment [0-9A-F]+ has already been removed"), 'check that required WAL segments are still available'); # Advance WAL again, the slot loses the oldest segment. -my $logstart = get_log_size($node_master); +my $logstart = $node_master->current_log_position(); advance_wal($node_master, 7); $node_master->safe_psql('postgres', "CHECKPOINT;"); # WARNING should be issued -ok( find_in_log( - $node_master, +ok( $node_master->find_in_log( "invalidating slot \"rep1\" because its restart_lsn [0-9A-F/]+ exceeds max_slot_wal_keep_size", $logstart), 'check that the warning is logged'); @@ -190,14 +188,13 @@ is($result, "rep1|f|t|lost|", 'check that the slot became inactive and the state "lost" persists'); # The standby no longer can connect to the master -$logstart = get_log_size($node_standby); +$logstart = $node_standby->current_log_position(); $node_standby->start; my $failed = 0; for (my $i = 0; $i < 10000; $i++) { - if (find_in_log( - $node_standby, + if ($node_standby->find_in_log( "requested WAL segment [0-9A-F]+ has already been removed", $logstart)) { @@ -264,25 +261,3 @@ sub advance_wal } return; } - -# return the size of logfile of $node in bytes -sub get_log_size -{ - my ($node) = @_; - - return (stat $node->logfile)[7]; -} - -# find $pat in logfile of $node after $off-th byte -sub find_in_log -{ - my ($node, $pat, $off) = @_; - - $off = 0 unless defined $off; - my $log = TestLib::slurp_file($node->logfile); - return 0 if (length($log) <= $off); - - $log = substr($log, $off); - - return $log =~ m/$pat/; -} -- 2.27.0 ----Next_Part(Wed_Jan__6_10_48_04_2021_104)---- ^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: Emitting JSON to file using COPY TO @ 2024-02-02 08:25 jian he <[email protected]> 0 siblings, 1 reply; 4+ messages in thread From: jian he @ 2024-02-02 08:25 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Joe Conway <[email protected]>; Daniel Verite <[email protected]>; Andrew Dunstan <[email protected]>; Davin Shearer <[email protected]>; pgsql-hackers; Junwang Zhao <[email protected]> On Wed, Jan 31, 2024 at 9:26 PM Alvaro Herrera <[email protected]> wrote: > > On 2024-Jan-23, jian he wrote: > > > > + | FORMAT_LA copy_generic_opt_arg > > > + { > > > + $$ = makeDefElem("format", $2, @1); > > > + } > > > ; > > > > > > I think it's not necessary. "format" option is already handled in > > > copy_generic_opt_elem. > > > > test it, I found out this part is necessary. > > because a query with WITH like `copy (select 1) to stdout with > > (format json, force_array false); ` will fail. > > Right, because "FORMAT JSON" is turned into FORMAT_LA JSON by parser.c > (see base_yylex there). I'm not really sure but I think it might be > better to make it "| FORMAT_LA JSON" instead of invoking the whole > copy_generic_opt_arg syntax. Not because of performance, but just > because it's much clearer what's going on. > sorry to bother you. Now I didn't apply any patch, just at the master. I don't know much about gram.y. copy (select 1) to stdout with (format json1); ERROR: COPY format "json1" not recognized LINE 1: copy (select 1) to stdout with (format json1); ^ copy (select 1) to stdout with (format json); ERROR: syntax error at or near "format" LINE 1: copy (select 1) to stdout with (format json); ^ json is a keyword. Is it possible to escape it? make `copy (select 1) to stdout with (format json)` error message the same as `copy (select 1) to stdout with (format json1)` ^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: Emitting JSON to file using COPY TO @ 2024-02-02 09:47 Alvaro Herrera <[email protected]> parent: jian he <[email protected]> 0 siblings, 1 reply; 4+ messages in thread From: Alvaro Herrera @ 2024-02-02 09:47 UTC (permalink / raw) To: jian he <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Joe Conway <[email protected]>; Daniel Verite <[email protected]>; Andrew Dunstan <[email protected]>; Davin Shearer <[email protected]>; pgsql-hackers; Junwang Zhao <[email protected]> On 2024-Feb-02, jian he wrote: > copy (select 1) to stdout with (format json); > ERROR: syntax error at or near "format" > LINE 1: copy (select 1) to stdout with (format json); > ^ > > json is a keyword. Is it possible to escape it? > make `copy (select 1) to stdout with (format json)` error message the same as > `copy (select 1) to stdout with (format json1)` Sure, you can use copy (select 1) to stdout with (format "json"); and then you get ERROR: COPY format "json" not recognized is that what you meant? If you want the server to send this message when the JSON word is not in quotes, I'm afraid that's not possible, due to the funny nature of the FORMAT keyword when the JSON keyword appears after it. But why do you care? If you use the patch, then you no longer need to have the "not recognized" error messages anymore, because the JSON format is indeed a recognized one. Maybe I didn't understand your question. -- Álvaro Herrera 48°01'N 7°57'E — https://www.EnterpriseDB.com/ ^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: Emitting JSON to file using COPY TO @ 2024-02-02 10:05 jian he <[email protected]> parent: Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: jian he @ 2024-02-02 10:05 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: Masahiko Sawada <[email protected]>; Joe Conway <[email protected]>; Daniel Verite <[email protected]>; Andrew Dunstan <[email protected]>; Davin Shearer <[email protected]>; pgsql-hackers; Junwang Zhao <[email protected]> On Fri, Feb 2, 2024 at 5:48 PM Alvaro Herrera <[email protected]> wrote: > > If you want the server to send this message when the JSON word is not in > quotes, I'm afraid that's not possible, due to the funny nature of the > FORMAT keyword when the JSON keyword appears after it. But why do you > care? If you use the patch, then you no longer need to have the "not > recognized" error messages anymore, because the JSON format is indeed > a recognized one. > "JSON word is not in quotes" is my intention. Now it seems when people implement any custom format for COPY, if the format_name is a keyword then we need single quotes. Thanks for clarifying! ^ permalink raw reply [nested|flat] 4+ messages in thread
end of thread, other threads:[~2024-02-02 10:05 UTC | newest] Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-01-05 04:34 [PATCH] Fix timeline-tracking failure while sending a historic timeline Kyotaro Horiguchi <[email protected]> 2024-02-02 08:25 Re: Emitting JSON to file using COPY TO jian he <[email protected]> 2024-02-02 09:47 ` Re: Emitting JSON to file using COPY TO Alvaro Herrera <[email protected]> 2024-02-02 10:05 ` Re: Emitting JSON to file using COPY TO jian he <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox