From: Andrew Gierth <andrew@tao11.riddles.org.uk>
To: pgsql-bugs@postgresql.org
Cc: Andres Freund <andres@anarazel.de>
Subject: Re: BUG #14057: vacuum setting reltuples=0 for tables with >0 tuples
Date: Thu, 16 Mar 2017 06:55:54 +0000
Message-ID: <87y3w53est.fsf@news-spur.riddles.org.uk> (raw)
In-Reply-To: <20160331104129.GG23562@awork2.anarazel.de>
References: <20160331103739.8956.94469@wrigleys.postgresql.org>
<20160331104129.GG23562@awork2.anarazel.de>
List-Unsubscribe: <mailto:majordomo@postgresql.org?body=unsub%20pgsql-bugs>
>>>>> "Andres" == Andres Freund <andres@anarazel.de> writes:
>> It looks like fixing this requires breaking scanned_pages out into
>> at least two separate counters, since we currently use it to track
>> both whether we can update stats, and whether we can update
>> relfrozenxid.
Andres> Are you planning to submit a fix?
Somewhat belatedly, yes.
Attached are patches against master and all the back branches back to
9.2. I've reproduced the bug on all of them, and confirmed that this
fixes it on all of them. Is it worth also including the isolation
tester script in the changes?
I can go ahead and commit these if they look ok.
--
Andrew (irc:RhodiumToad)
--
Sent via pgsql-bugs mailing list (pgsql-bugs@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-bugsAttachments:download all (6 files, .tar.gz)vacstats_92.patch (text/x-patch, 3.1K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex 8aefa7aaa9..a33541a115 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -100,7 +100,8 @@ typedef struct LVRelStats
BlockNumber old_rel_pages; /* previous value of pg_class.relpages */
BlockNumber rel_pages; /* total number of pages */
BlockNumber scanned_pages; /* number of pages we examined */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
BlockNumber pages_removed;
@@ -258,6 +259,10 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not get+ * cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -267,7 +272,7 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -423,6 +428,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -613,6 +619,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -999,7 +1006,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1264,7 +1271,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
vacstats_93.patch (text/x-patch, 3.1K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex 86114f7b89..6ece3591e4 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -103,7 +103,8 @@ typedef struct LVRelStats
BlockNumber old_rel_pages; /* previous value of pg_class.relpages */
BlockNumber rel_pages; /* total number of pages */
BlockNumber scanned_pages; /* number of pages we examined */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
BlockNumber pages_removed;
@@ -278,6 +279,10 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not get+ * cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -287,7 +292,7 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -446,6 +451,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -636,6 +642,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -1046,7 +1053,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1350,7 +1357,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
vacstats_94.patch (text/x-patch, 3.1K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex 50085f9a54..95f5952f63 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -104,7 +104,8 @@ typedef struct LVRelStats
BlockNumber old_rel_pages; /* previous value of pg_class.relpages */
BlockNumber rel_pages; /* total number of pages */
BlockNumber scanned_pages; /* number of pages we examined */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
double new_dead_tuples; /* new estimated total # of dead tuples */
@@ -281,6 +282,10 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not get+ * cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -290,7 +295,7 @@ lazy_vacuum_rel(Relation onerel, VacuumStmt *vacstmt,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -455,6 +460,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -645,6 +651,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -1053,7 +1060,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1357,7 +1364,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
vacstats_95.patch (text/x-patch, 3.1K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex 64626edf70..2654c9ebef 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -106,7 +106,8 @@ typedef struct LVRelStats
BlockNumber rel_pages; /* total number of pages */
BlockNumber scanned_pages; /* number of pages we examined */
BlockNumber pinskipped_pages; /* # of pages we skipped due to a pin */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
double new_dead_tuples; /* new estimated total # of dead tuples */
@@ -286,6 +287,10 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not get+ * cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -295,7 +300,7 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -474,6 +479,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -666,6 +672,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -1074,7 +1081,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1393,7 +1400,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
vacstats_96.patch (text/x-patch, 3.2K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex b5fb325007..56311fef75 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -108,7 +108,8 @@ typedef struct LVRelStats
BlockNumber scanned_pages; /* number of pages we examined */
BlockNumber pinskipped_pages; /* # of pages we skipped due to a pin */
BlockNumber frozenskipped_pages; /* # of frozen pages we skipped */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
double new_dead_tuples; /* new estimated total # of dead tuples */
@@ -293,6 +294,10 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not get+ * cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -302,7 +307,7 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -489,6 +494,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -811,6 +817,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -1254,7 +1261,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1618,7 +1625,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
vacstats_head.patch (text/x-patch, 3.2K) downloaddiff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.cindex 5d47f16ee4..b74e4934ec 100644--- a/src/backend/commands/vacuumlazy.c+++ b/src/backend/commands/vacuumlazy.c@@ -114,7 +114,8 @@ typedef struct LVRelStats
BlockNumber scanned_pages; /* number of pages we examined */
BlockNumber pinskipped_pages; /* # of pages we skipped due to a pin */
BlockNumber frozenskipped_pages; /* # of frozen pages we skipped */
- double scanned_tuples; /* counts only tuples on scanned pages */+ BlockNumber tupcount_pages; /* pages whose tuples we counted */+ double scanned_tuples; /* counts only tuples on tupcount_pages */
double old_rel_tuples; /* previous value of pg_class.reltuples */
double new_rel_tuples; /* new estimated total # of tuples */
double new_dead_tuples; /* new estimated total # of dead tuples */
@@ -299,6 +300,10 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
* density") with nonzero relpages and reltuples=0 (which means "zero
* tuple density") unless there's some actual evidence for the latter.
*
+ * It's important that we use tupcount_pages and not scanned_pages for the+ * check described above; scanned_pages counts pages where we could not+ * get cleanup lock, and which were processed only for frozenxid purposes.+ *
* We do update relallvisible even in the corner case, since if the table
* is all-visible we'd definitely like to know that. But clamp the value
* to be not more than what we're setting relpages to.
@@ -308,7 +313,7 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params,
*/
new_rel_pages = vacrelstats->rel_pages;
new_rel_tuples = vacrelstats->new_rel_tuples;
- if (vacrelstats->scanned_pages == 0 && new_rel_pages > 0)+ if (vacrelstats->tupcount_pages == 0 && new_rel_pages > 0)
{
new_rel_pages = vacrelstats->old_rel_pages;
new_rel_tuples = vacrelstats->old_rel_tuples;
@@ -496,6 +501,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
nblocks = RelationGetNumberOfBlocks(onerel);
vacrelstats->rel_pages = nblocks;
vacrelstats->scanned_pages = 0;
+ vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
vacrelstats->latestRemovedXid = InvalidTransactionId;
@@ -818,6 +824,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
}
vacrelstats->scanned_pages++;
+ vacrelstats->tupcount_pages++;
page = BufferGetPage(buf);
@@ -1261,7 +1268,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
/* now we can compute the new value for pg_class.reltuples */
vacrelstats->new_rel_tuples = vac_estimate_reltuples(onerel, false,
nblocks,
- vacrelstats->scanned_pages,+ vacrelstats->tupcount_pages,
num_tuples);
/*
@@ -1625,7 +1632,7 @@ lazy_cleanup_index(Relation indrel,
ivinfo.index = indrel;
ivinfo.analyze_only = false;
- ivinfo.estimated_count = (vacrelstats->scanned_pages < vacrelstats->rel_pages);+ ivinfo.estimated_count = (vacrelstats->tupcount_pages < vacrelstats->rel_pages);
ivinfo.message_level = elevel;
ivinfo.num_heap_tuples = vacrelstats->new_rel_tuples;
ivinfo.strategy = vac_strategy;
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-bugs@postgresql.org
Cc: andrew@tao11.riddles.org.uk, andres@anarazel.de
Subject: Re: BUG #14057: vacuum setting reltuples=0 for tables with >0 tuples
In-Reply-To: <87y3w53est.fsf@news-spur.riddles.org.uk>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox