From: Stas Kelvich <s.kelvich@postgrespro.ru>
To: John Naylor <jcnaylor@gmail.com>
Cc: andreas@proxel.se, Tom Lane <tgl@sss.pgh.pa.us>
Cc: Andrew Dunstan <andrew.dunstan@2ndquadrant.com>
Cc: PostgreSQL Hackers <pgsql-hackers@postgresql.org>
Cc: David Fetter <david@fetter.org>
Subject: Re: unused_oids script is broken with bsd sed
Date: Wed, 25 Apr 2018 20:55:08 +0300
Message-ID: <26728623-7783-4103-9BED-08D443C1729F@postgrespro.ru> (raw)
In-Reply-To: <CAJVSVGU2w6rvTXghDe7-Y3x9Ca0833VmnC3HpaYKGLw4xznpJA@mail.gmail.com>
References: <37D774E4-FE1F-437E-B3D2-593F314B7505@postgrespro.ru>
<34a54c54-9132-1254-2bc3-d3489fb79728@2ndQuadrant.com>
<4473.1524665917@sss.pgh.pa.us>
<4DE444D1-DDE9-460F-AC2F-EB1B73D94A2D@postgrespro.ru>
<CAJVSVGU2w6rvTXghDe7-Y3x9Ca0833VmnC3HpaYKGLw4xznpJA@mail.gmail.com>
> On 25 Apr 2018, at 17:55, John Naylor <jcnaylor@gmail.com> wrote:
>
> On 4/25/18, Stas Kelvich <s.kelvich@postgrespro.ru> wrote:
>>> On 25 Apr 2018, at 17:18, Tom Lane <tgl@sss.pgh.pa.us> wrote:
>>> I think we should rewrite
>>> both of them to use the Catalog.pm infrastructure.
>>
>> Okay, seems reasonable. I'll put shared code in Catalog.pm and
>> update patch.
>
> I don't think you need any new code in Catalog.pm, I believe the
> suggestion was just to use that module as a stable interface to the
> data. Looking at your patch, I'll mention that we have an idiom for
> extracting #define'd OID symbols, e.g.:
>
> my $FirstBootstrapObjectId = Catalog::FindDefinedSymbol(
> 'access/transam.h', \@include_path, 'FirstBootstrapObjectId');
>
> This is preferred over using awk, which would have its own portability
> issues (Windows for starters).
>
> While I'm thinking out loud, it might be worthwhile to patch genbki.pl
> for the duplicate test, since they're run at the same time anyway (see
> catalog/Makefile), and we've already read all the data.
>
> -John Naylor
>
New version is attached. I've put iterator into Catalog.pm to avoid copy-pasting
it over two scripts. Also instead of greping through *.dat and *.h files I've
used parsers provided in Catalog module. That way should be more sustainable
to format changes.
Anyone who wanted to help with scripts -- feel free to rewrite.
--
Stas Kelvich
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company
Attachments:0001-Rewrite-unused_oids-in-perl.patch (application/octet-stream, 6.0K) download
From a86a01be7f35d2f71cdfc3e887d29e2d5edf820a Mon Sep 17 00:00:00 2001
From: Stas Kelvich <stanconn@gmail.com>
Date: Wed, 25 Apr 2018 13:00:28 +0300
Subject: [PATCH] Rewrite unused_oids in perl
unused_oids script was broken with bsd sed after 372728b0d49. It was
possible to write more portable regexps but it seems easier to just rewrite
unused_oids in perl to match duplicate_oids. Also add in-place complain
about duplicates instead of running uniq through oids array.
---
src/backend/catalog/Catalog.pm | 47 ++++++++++++++++++++++++++++
src/include/catalog/duplicate_oids | 23 +++++---------
src/include/catalog/unused_oids | 63 ++++++++++++++++++--------------------
3 files changed, 85 insertions(+), 48 deletions(-)
diff --git a/src/backend/catalog/Catalog.pm b/src/backend/catalog/Catalog.pmindex 6305a2b362..5caf428b6a 100644--- a/src/backend/catalog/Catalog.pm+++ b/src/backend/catalog/Catalog.pm@@ -40,7 +40,9 @@ sub ParseHeader
$catalog{columns} = [];
$catalog{toasting} = [];
+ $catalog{toasting_oids} = [];
$catalog{indexing} = [];
+ $catalog{indexing_oids} = [];
$catalog{client_code} = [];
open(my $ifh, '<', $input_file) || die "$input_file: $!";
@@ -89,6 +91,7 @@ sub ParseHeader
my ($toast_name, $toast_oid, $index_oid) = ($1, $2, $3);
push @{ $catalog{toasting} },
"declare toast $toast_oid $index_oid on $toast_name\n";
+ push @{ $catalog{toasting_oids} }, ($toast_oid, $index_oid);
}
elsif (/^DECLARE_(UNIQUE_)?INDEX\(\s*(\w+),\s*(\d+),\s*(.+)\)/)
{
@@ -99,6 +102,7 @@ sub ParseHeader
"declare %sindex %s %s %s\n",
$is_unique ? 'unique ' : '',
$index_name, $index_oid, $using);
+ push @{ $catalog{indexing_oids} }, ($index_oid);
}
elsif (/^BUILD_INDICES/)
{
@@ -389,4 +393,47 @@ sub FindDefinedSymbolFromData
die "no definition found for $symbol\n";
}
+sub FindAllOidsFromHeaders+{+ my @input_files = @_;+ my @oids = ();++ foreach my $header (@input_files)+ {+ $header =~ /(.+)\.h$/+ or die "Input files need to be header files.\n";+ my $datfile = "$1.dat";++ my $catalog = Catalog::ParseHeader($header);++ push @oids, $catalog->{rowtype_oid} if ($catalog->{rowtype_oid});+ push @oids, $catalog->{relation_oid} if ($catalog->{relation_oid});++ # pg_class row oids are duplicated in typrelid of pg_type+ if (exists($catalog->{catname}) and ($catalog->{catname} eq 'pg_class'))+ {+ next;+ }++ if (-e $datfile)+ {+ my $catdata = Catalog::ParseData($datfile, $catalog->{columns}, 0);+ foreach my $row (@$catdata)+ {+ # filter entries for bootstrapped catalogs entries in pg_type+ if (($catalog->{catname} eq 'pg_type') and ($row->{typcategory} eq 'C'))+ {+ next;+ }+ push @oids, $row->{oid} if defined $row->{oid};+ }+ }++ push @oids, @{ $catalog->{toasting_oids} };+ push @oids, @{ $catalog->{indexing_oids} };+ }++ return \@oids;+}+
1;
diff --git a/src/include/catalog/duplicate_oids b/src/include/catalog/duplicate_oidsindex 8c143cf06f..c818dc8a67 100755--- a/src/include/catalog/duplicate_oids+++ b/src/include/catalog/duplicate_oids@@ -1,27 +1,20 @@
#!/usr/bin/perl
+use lib '../../backend/catalog/';+use Catalog;+
use strict;
use warnings;
-BEGIN-{- @ARGV = (glob("pg_*.h"), glob("pg_*.dat"), qw(indexing.h toasting.h));-}+my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h));++my @oids = @{ Catalog::FindAllOidsFromHeaders(@input_files) };
my %oidcounts;
-while (<>)+foreach my $oid (@oids)
{
- next if /^CATALOG\(.*BKI_BOOTSTRAP/;- next- unless /\boid *=> *'(\d+)'/- || /^CATALOG\([^,]*, *(\d+).*BKI_ROWTYPE_OID\((\d+),/- || /^CATALOG\([^,]*, *(\d+)/- || /^DECLARE_INDEX\([^,]*, *(\d+)/- || /^DECLARE_UNIQUE_INDEX\([^,]*, *(\d+)/- || /^DECLARE_TOAST\([^,]*, *(\d+), *(\d+)/;- $oidcounts{$1}++;- $oidcounts{$2}++ if $2;+ $oidcounts{$oid}++;
}
my $found = 0;
diff --git a/src/include/catalog/unused_oids b/src/include/catalog/unused_oidsindex f71222d50d..4b0e16cc02 100755--- a/src/include/catalog/unused_oids+++ b/src/include/catalog/unused_oids@@ -1,4 +1,4 @@-#!/bin/sh+#!/usr/bin/perl
#
# unused_oids
#
@@ -15,43 +15,40 @@
#
# run this script in src/include/catalog.
#
+use lib '../../backend/catalog/';+use Catalog;+use strict;+use warnings;-AWK="awk"+my @input_files = (glob("pg_*.h"), qw(indexing.h toasting.h));+my @oids;-# Get FirstBootstrapObjectId from access/transam.h-FIRSTOBJECTID=`grep '#define[ ]*FirstBootstrapObjectId' ../access/transam.h | $AWK '{ print $3 }'`-export FIRSTOBJECTID+push @oids, @{ Catalog::FindAllOidsFromHeaders(@input_files) };-# this part (down to the uniq step) should match the duplicate_oids script-# note: we exclude BKI_BOOTSTRAP relations since they are expected to have-# matching data entries in pg_class.dat and pg_type.dat+my $FirstBootstrapObjectId = Catalog::FindDefinedSymbol(+ 'access/transam.h', [".."], 'FirstBootstrapObjectId');+push @oids, $FirstBootstrapObjectId;-cat pg_*.h pg_*.dat toasting.h indexing.h |-egrep -v -e '^CATALOG\(.*BKI_BOOTSTRAP' | \-sed -n -e 's/.*\boid *=> *'\''\([0-9][0-9]*\)'\''.*$/\1/p' \- -e 's/^CATALOG([^,]*, *\([0-9][0-9]*\).*BKI_ROWTYPE_OID(\([0-9][0-9]*\),.*$/\1,\2/p' \- -e 's/^CATALOG([^,]*, *\([0-9][0-9]*\).*$/\1/p' \- -e 's/^DECLARE_INDEX([^,]*, *\([0-9][0-9]*\).*$/\1/p' \- -e 's/^DECLARE_UNIQUE_INDEX([^,]*, *\([0-9][0-9]*\).*$/\1/p' \- -e 's/^DECLARE_TOAST([^,]*, *\([0-9][0-9]*\), *\([0-9][0-9]*\).*$/\1,\2/p' | \-tr ',' '\n' | \-sort -n | \-uniq | \-$AWK '-BEGIN {- last = 0;-}-/^[0-9]/ {- if ($1 > last + 1) {- if ($1 > last + 2) {- print last + 1, "-", $1 - 1;- } else {- print last + 1;++my $prev_oid = 0;+foreach my $oid (sort { $a <=> $b } @oids)+{+ if ($oid > $prev_oid + 1)+ {+ if ($oid > $prev_oid + 2)+ {+ print "@{[ $prev_oid + 1 ]} - @{[ $oid - 1 ]}\n";
}
+ else+ {+ print "@{[ $prev_oid + 1 ]}\n";+ }+ }+ elsif ($oid == $prev_oid)+ {+ print "Duplicate oid detected: $oid\n";+ exit 1;
}
- last = $1;+ $prev_oid = $oid;
}
-END {- print last + 1, "-", ENVIRON["FIRSTOBJECTID"]-1;-}'--
2.16.2
=
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: s.kelvich@postgrespro.ru, jcnaylor@gmail.com, tgl@sss.pgh.pa.us, andrew.dunstan@2ndquadrant.com, david@fetter.org
Subject: Re: unused_oids script is broken with bsd sed
In-Reply-To: <26728623-7783-4103-9BED-08D443C1729F@postgrespro.ru>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox