Date: Sun, 7 Nov 2021 15:22:02 -0800
From: Noah Misch <noah@leadboat.com>
To: Semab Tariq <semab.tariq@enterprisedb.com>
Cc: Thomas Munro <thomas.munro@gmail.com>,
	Sandeep Thakkar <sandeep.thakkar@enterprisedb.com>,
	Robert Haas <robertmhaas@gmail.com>,
	Andrey Borodin <x4mmm@yandex-team.ru>,
	Andres Freund <andres@anarazel.de>, CM Team <cm@enterprisedb.com>,
	PostgreSQL mailing lists <pgsql-bugs@lists.postgresql.org>,
	Michael Paquier <michael@paquier.xyz>, Tom Lane <tgl@sss.pgh.pa.us>,
	Peter Geoghegan <pg@bowt.ie>
Subject: Re: CREATE INDEX CONCURRENTLY does not index prepared xact's data
Message-ID: <20211107232202.GA882747@rfd.leadboat.com>
References: <20211103044203.GA570491@rfd.leadboat.com>
 <CABimMB4mRs9N3eivR-=qF9M8oWc5E6OX7GywsWF0DXN4P5gNEA@mail.gmail.com>
 <CA+hUKG+67HCS6sJta2xiaf3Fbw+wx_jyGT+E7nKXWqK5zNyJ7Q@mail.gmail.com>
 <20211105013553.GA671366@rfd.leadboat.com>
 <CABimMB4RYDOtVKLp34U7n316ARr4cBBX0dxEgEXMA7X-Coh2=Q@mail.gmail.com>
 <20211105090347.GB671366@rfd.leadboat.com>
 <CABimMB6HFAu3fRn3sD0EKU=Hhz5fxQNixrhB__8Wk-p7v5zExg@mail.gmail.com>
 <20211105112236.GA711250@rfd.leadboat.com>
 <20211106230846.GA790288@rfd.leadboat.com>
 <CABimMB4dQXvhB2Ja93BAWJzpM89RDzF-vdpM+=s4W7EbN+jKDw@mail.gmail.com>
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="EVF5PPMfhYS0aIcm"
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
In-Reply-To: 
 <CABimMB4dQXvhB2Ja93BAWJzpM89RDzF-vdpM+=s4W7EbN+jKDw@mail.gmail.com>
User-Agent: Mutt/1.5.24 (2015-08-30)
Archived-At: 
 <https://www.postgresql.org/message-id/20211107232202.GA882747%40rfd.leadboat.com>
Precedence: bulk


--EVF5PPMfhYS0aIcm
Content-Type: text/plain; charset=iso-8859-1
Content-Disposition: inline
Content-Transfer-Encoding: 8bit

On Sun, Nov 07, 2021 at 08:25:09PM +0500, Semab Tariq wrote:
> On Sun, Nov 7, 2021 at 4:08 AM Noah Misch <noah@leadboat.com> wrote:
> > On Fri, Nov 05, 2021 at 04:22:36AM -0700, Noah Misch wrote:
> > > On Fri, Nov 05, 2021 at 03:21:15PM +0500, Semab Tariq wrote:
> > > > Breakpoint 1, 0x40000000003fcb50:0 in equalTupleDescs (
> > > >     tupdesc1=0x40010006f968, tupdesc2=0x87ffffffffffac50)
> > >
> > > The addresses there are weird.  tupdesc1 is neither a stack address nor a heap
> > > address; it may be in a program text section.  tupdesc2 is a stack address.
> > > In the earlier stack trace from
> > > https://postgr.es/m/CANFyU94Xa8a5+4sZ7PxOiDLq+yN89g6y-9nNk-eLEvX6YUXbXA@mail.gmail.com
> > > both tupdesc1 and tupdesc2 were heap addresses.

That turned out to be a false alarm.  On gharial, a breakpoint at the start of
the function doesn't see the real arguments.  After a ten-instruction
prologue, the real arguments appear, and they are heap addresses.

> PFA new regress_log_080_step_equalTupleDescs file generated from your latest patch

Thanks.  That shows the crash happened sometime after strcmp(defval1->adbin,
defval2->adbin).  Please run the attached version, which collects yet more
information.

--EVF5PPMfhYS0aIcm
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="step-to-crash-v0.4.patch"

Author:     Noah Misch <noah@leadboat.com>
Commit:     Noah Misch <noah@leadboat.com>

    
diff --git a/src/test/subscription/t/080_step_equalTupleDescs.pl b/src/test/subscription/t/080_step_equalTupleDescs.pl
new file mode 100644
index 0000000..a4dec82
--- /dev/null
+++ b/src/test/subscription/t/080_step_equalTupleDescs.pl
@@ -0,0 +1,223 @@
+# run with: make -C src/test/subscription check PROVE_TESTS=t/080_step_equalTupleDescs.pl
+# email src/test/subscription/tmp_check/log/regress_log_080_step_equalTupleDescs
+
+use strict;
+use warnings;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 4;
+use IPC::Run;
+use Time::HiRes qw(usleep);
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+$node->init;
+my $gdb_start_witness = $node->basedir . '/gdb_started.txt';
+
+$ENV{PATH} =~ /^([^:]*):/;
+my $bindir = $1;
+die "could not guess bindir from PATH=$ENV{PATH}" unless $1;
+
+# I seem to recall gdb vintages where different attach methods had different
+# ability to find source code.  Try that.
+my @attach_methods = (
+	# --pid alone
+	sub {
+		my $pid = shift;
+		return (
+			['--pid', $pid],
+			'');
+	},
+	# --pid w/ explicit symbol-file
+	sub {
+		my $pid = shift;
+		return (
+			['--pid', $pid],
+			"symbol-file -readnow $bindir/postgres");
+	},
+	# "attach" alone
+	sub {
+		my $pid = shift;
+		return (
+			[],
+			"attach $pid");
+	},
+	# "attach" w/ explicit symbol-file
+	sub {
+		my $pid = shift;
+		return (
+			[],
+			"symbol-file -readnow $bindir/postgres\nattach $pid");
+	},
+);
+
+foreach my $method (@attach_methods)
+{
+	poll_start($node);
+	$node->safe_psql('postgres', 'DROP TABLE IF EXISTS attmp');
+	unlink $gdb_start_witness;
+
+	my $in    = '';
+	my $out   = '';
+	my $timer = IPC::Run::timeout(180);
+	my $psql  = $node->background_psql('postgres', \$in, \$out, $timer,
+		on_error_stop => 0);
+
+	$in .= q{
+SELECT pg_backend_pid(), 'after_pid';
+};
+	$psql->pump until $out =~ /^([0-9]*)\|after_pid/m;
+	my $pid = $1;
+	print "backend pid: $pid\n";
+
+	my($attach_args, $attach_cmd) = $method->($pid);
+	my $gdb_in = '';
+	my $gdb_cmdline = [ qw(gdb --nx), @$attach_args, "$bindir/postgres" ];
+	print("# Running: " . join(" ", @{$gdb_cmdline}) . "\n");
+	my $gdb = IPC::Run::start($gdb_cmdline, '<', \$gdb_in, $timer);
+
+	print("# Using attach command: $attach_cmd\n");
+	$gdb_in = q{
+set width 0
+#
+} . $attach_cmd . q{
+#
+b equalTupleDescs
+ignore 1 9
+#
+set logging file } . $gdb_start_witness . q{
+set logging on
+set logging off
+#
+echo disassembly attempts, not sure which will work on HP gdb 6.1\n
+disassemble /m equalTupleDescs
+disassemble 0x40000000003fcb50,0x40000000003fdfff
+disassemble 0x40000000003fcb50 0x40000000003fdfff
+echo source listing attempts -\
+ try it, but gdb is seeing line numbers for main.c and toasting.c only\n
+list *0x40000000003fdc30
+list *0x40000000003fdc30:2
+};
+	$gdb->pump until $gdb_in eq '';
+	sleep .1 while !-f $gdb_start_witness;
+
+	# first 28 lines of src/test/regress/sql/alter_table.sql
+	$in .= q{
+--
+-- ALTER_TABLE
+--
+
+-- Clean up in case a prior regression run failed
+SET client_min_messages TO 'warning';
+DROP ROLE IF EXISTS regress_alter_table_user1;
+RESET client_min_messages;
+
+CREATE USER regress_alter_table_user1;
+
+--
+-- add attribute
+--
+
+CREATE TABLE attmp (initial int4);
+
+COMMENT ON TABLE attmp_wrong IS 'table comment';
+COMMENT ON TABLE attmp IS 'table comment';
+COMMENT ON TABLE attmp IS NULL;
+
+ALTER TABLE attmp ADD COLUMN xmin integer; -- fails
+
+ALTER TABLE attmp ADD COLUMN a int4 default 3;
+
+ALTER TABLE attmp ADD COLUMN b name;
+
+ALTER TABLE attmp ADD COLUMN c text;
+};
+
+	$in .= q{
+\q
+};
+
+	$gdb_in .= q{
+echo Reach breakpoint.\n
+continue
+#
+echo On gharial, args are properly setup after a 10-instr preamble.  Advance\
+ 11 instructions (in case I counted wrong) before capturing data.\n
+set $loopvar = 11
+while ($loopvar--)
+  stepi
+  x/i $pc
+end
+#
+echo Print any interesting memory state.\n
+p *tupdesc1
+p *tupdesc2
+p tupdesc1->attrs[0]
+p tupdesc1->attrs[1]
+p tupdesc1->attrs[2]
+p tupdesc2->attrs[0]
+p tupdesc2->attrs[1]
+p tupdesc2->attrs[2]
+p *tupdesc1->constr
+p *tupdesc2->constr
+p tupdesc1->constr->defval[0]
+p tupdesc2->constr->defval[0]
+backtrace full
+info proc mappings
+set $loopvar = 25
+while ($loopvar--)
+  frame $loopvar
+  info frame
+end
+#
+echo Advance to SIGSEGV, or give up after 25000 instructions.\n
+set $loopvar = 25000
+while ($loopvar--)
+  stepi
+  x/i $pc
+end
+maintenance info sections ALLOBJ
+info proc mappings
+quit
+};
+
+	for (;;)
+	{
+		$psql->pump_nb;
+		$gdb->pump_nb;
+		last if $in eq '' && $gdb_in eq '';
+		sleep .1;
+	}
+
+	$psql->finish;
+	$gdb->finish;
+	ok(1);
+
+	# Like $node->kill9, but don't care if a crash already downed the node.
+	kill(9, $node->{_pid});
+	$node->{_pid} = undef;
+}
+
+
+# copied from 017_shm.pl
+sub poll_start
+{
+	my ($node) = @_;
+
+	my $max_attempts = 180 * 10;
+	my $attempts     = 0;
+
+	while ($attempts < $max_attempts)
+	{
+		$node->start(fail_ok => 1) && return 1;
+
+		# Wait 0.1 second before retrying.
+		usleep(100_000);
+
+		$attempts++;
+	}
+
+	# No success within 180 seconds.  Try one last time without fail_ok, which
+	# will BAIL_OUT unless it succeeds.
+	$node->start && return 1;
+	return 0;
+}

--EVF5PPMfhYS0aIcm--