agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH] Fix misuses of RelationNeedsWAL 9+ messages / 2 participants [nested] [flat]
* [PATCH] Fix misuses of RelationNeedsWAL @ 2021-01-13 06:52 Kyotaro Horiguchi <horikyoga.ntt@gmail.com> 0 siblings, 0 replies; 9+ messages in thread From: Kyotaro Horiguchi @ 2021-01-13 06:52 UTC (permalink / raw) The definition of the macro RelationNeedsWAL has been changed by c6b92041d3 to include conditions related to the WAL-skip optimization but some uses of the macro are not relevant to the optimization. That misuses are harmless for now as they are only executed while wal_level >= replica or WAL-skipping is inactive. However, this should be corrected to prevent future hazard. --- src/backend/catalog/pg_publication.c | 2 +- src/backend/optimizer/util/plancat.c | 2 +- src/include/utils/rel.h | 15 +++++++++++---- src/include/utils/snapmgr.h | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/backend/catalog/pg_publication.c b/src/backend/catalog/pg_publication.c index 5f8e1c64e1..f3060a4cf1 100644 --- a/src/backend/catalog/pg_publication.c +++ b/src/backend/catalog/pg_publication.c @@ -67,7 +67,7 @@ check_publication_add_relation(Relation targetrel) errdetail("System tables cannot be added to publications."))); /* UNLOGGED and TEMP relations cannot be part of publication. */ - if (!RelationNeedsWAL(targetrel)) + if (!RelationIsWalLogged(targetrel)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("table \"%s\" cannot be replicated", diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index da322b453e..0500efcdb9 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -126,7 +126,7 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, relation = table_open(relationObjectId, NoLock); /* Temporary and unlogged relations are inaccessible during recovery. */ - if (!RelationNeedsWAL(relation) && RecoveryInProgress()) + if (!RelationIsWalLogged(relation) && RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot access temporary or unlogged relations during recovery"))); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 10b63982c0..810806a542 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -552,16 +552,23 @@ typedef struct ViewOptions (relation)->rd_smgr->smgr_targblock = (targblock); \ } while (0) +/* + * RelationIsPermanent + * True if relation is WAL-logged. + */ +#define RelationIsWalLogged(relation) \ + ((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT) + /* * RelationNeedsWAL - * True if relation needs WAL. + * True if relation needs WAL at the time. * * Returns false if wal_level = minimal and this relation is created or * truncated in the current transaction. See "Skipping WAL for New * RelFileNode" in src/backend/access/transam/README. */ #define RelationNeedsWAL(relation) \ - ((relation)->rd_rel->relpersistence == RELPERSISTENCE_PERMANENT && \ + (RelationIsWalLogged(relation) && \ (XLogIsNeeded() || \ (relation->rd_createSubid == InvalidSubTransactionId && \ relation->rd_firstRelfilenodeSubid == InvalidSubTransactionId))) @@ -619,7 +626,7 @@ typedef struct ViewOptions */ #define RelationIsAccessibleInLogicalDecoding(relation) \ (XLogLogicalInfoActive() && \ - RelationNeedsWAL(relation) && \ + RelationIsWalLogged(relation) && \ (IsCatalogRelation(relation) || RelationIsUsedAsCatalogTable(relation))) /* @@ -635,7 +642,7 @@ typedef struct ViewOptions */ #define RelationIsLogicallyLogged(relation) \ (XLogLogicalInfoActive() && \ - RelationNeedsWAL(relation) && \ + RelationIsWalLogged(relation) && \ !IsCatalogRelation(relation)) /* routines in utils/cache/relcache.c */ diff --git a/src/include/utils/snapmgr.h b/src/include/utils/snapmgr.h index 579be352c5..7be922a9f1 100644 --- a/src/include/utils/snapmgr.h +++ b/src/include/utils/snapmgr.h @@ -37,7 +37,7 @@ */ #define RelationAllowsEarlyPruning(rel) \ ( \ - RelationNeedsWAL(rel) \ + RelationIsWalLogged(rel) \ && !IsCatalogRelation(rel) \ && !RelationIsAccessibleInLogicalDecoding(rel) \ ) -- 2.27.0 ----Next_Part(Wed_Jan_13_16_07_05_2021_621)---- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.8 27/38] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-14 16:16 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-14 16:16 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index b44cc358f29..0667efe31ca 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -422,6 +422,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --ow5flh3n247znjrs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.8-0028-aio-Add-test_aio-module.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.9 19/30] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-15 16:29 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-15 16:29 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index d7b395b86a3..887be2b3961 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -418,6 +418,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --23jbdfobqrqxnmx5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.9-0020-aio-Add-test_aio-module.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.11 18/27] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index 60a841816d8..facf996608e 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -418,6 +418,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --bjnmbpad43bpmfxt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.11-0019-aio-Implement-smgr-md-fd-write-support.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.13 18/28] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index 36c54fb695b..cec93129f58 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -404,6 +404,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --pro7bqageygxfsvg Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.13-0019-aio-Implement-smgr-md-fd-write-support.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.14 19/29] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index 36c54fb695b..cec93129f58 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -404,6 +404,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --m2na7lgr3zfazgom Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.14-0020-aio-Implement-smgr-md-fd-write-support.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.15 08/18] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index 36c54fb695b..cec93129f58 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -404,6 +404,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --xevce4sdbnyxplun Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.15-0009-aio-Implement-smgr-md-fd-write-support.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.12 18/28] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index f675168e89a..039d7dc71a5 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -403,6 +403,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --5i73spx2p4vwf7fe Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.12-0019-aio-Implement-smgr-md-fd-write-support.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2.10 16/28] aio: Experimental heuristics to increase batching in read_stream.c @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 9+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/backend/storage/aio/read_stream.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/backend/storage/aio/read_stream.c b/src/backend/storage/aio/read_stream.c index d7b395b86a3..887be2b3961 100644 --- a/src/backend/storage/aio/read_stream.c +++ b/src/backend/storage/aio/read_stream.c @@ -418,6 +418,30 @@ read_stream_start_pending_read(ReadStream *stream) static void read_stream_look_ahead(ReadStream *stream) { + /* + * Batch-submitting multiple IOs is more efficient than doing so + * one-by-one. If we just ramp up to the max, we'll only be allowed to + * submit one io_combine_limit sized IO. Defer submitting IO in that case. + * + * FIXME: This needs better heuristics. + */ +#if 1 + if (!stream->sync_mode && stream->distance > (io_combine_limit * 8)) + { + if (stream->pinned_buffers + stream->pending_read_nblocks > ((stream->distance * 3) / 4)) + { +#if 0 + ereport(LOG, + errmsg("reduce reduce reduce: pinned: %d, pending: %d, distance: %d", + stream->pinned_buffers, + stream->pending_read_nblocks, + stream->distance)); +#endif + return; + } + } +#endif + /* * Allow amortizing the cost of submitting IO over multiple IOs. This * requires that we don't do any operations that could lead to a deadlock -- 2.48.1.76.g4e746b1a31.dirty --w6dfit2y42fwvotd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.10-0017-aio-Add-test_aio-module.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2025-03-18 18:40 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-01-13 06:52 [PATCH] Fix misuses of RelationNeedsWAL Kyotaro Horiguchi <horikyoga.ntt@gmail.com> 2025-03-14 16:16 [PATCH v2.8 27/38] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-15 16:29 [PATCH v2.9 19/30] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.11 18/27] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.13 18/28] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.14 19/29] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.15 08/18] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.12 18/28] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.10 16/28] aio: Experimental heuristics to increase batching in read_stream.c Andres Freund <andres@anarazel.de>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox