agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH v2 14/20] aio: Add bounce buffers 23+ messages / 2 participants [nested] [flat]
* [PATCH v2 14/20] aio: Add bounce buffers @ 2024-11-25 21:35 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2024-11-25 21:35 UTC (permalink / raw) --- src/include/storage/aio.h | 18 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 182 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 118 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 419 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index ff44dac5bb2..1bef475b0a9 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -222,6 +222,9 @@ typedef struct PgAioHandleSharedCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + /* * How many callbacks can be registered for one IO handle. Currently we only * need two, but it's not hard to imagine needing a few more. @@ -294,6 +297,20 @@ extern void pgaio_result_log(PgAioResult result, const PgAioSubjectData *subject +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -354,6 +371,7 @@ typedef enum IoMethod extern const struct config_enum_entry io_method_options[]; extern int io_method; extern int io_max_concurrency; +extern int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index d2dc1516bdf..2065bde79c3 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -91,6 +91,12 @@ struct PgAioHandle /* index into PgAioCtl->iovecs */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -130,11 +136,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioPerBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -162,6 +180,12 @@ typedef struct PgAioPerBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioPerBackend; @@ -187,6 +211,15 @@ typedef struct PgAioCtl */ uint64 *iovecs_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 2d55720a54c..0cdd0c13ffb 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 893f4ffe428..0076ea4aa10 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -395,6 +395,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 2439ce3740d..e829e1752ca 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -54,6 +54,8 @@ static void pgaio_io_resowner_register(PgAioHandle *ioh); static void pgaio_io_wait_for_free(void); static PgAioHandle *pgaio_io_from_ref(PgAioHandleRef *ior, uint64 *ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ @@ -68,6 +70,7 @@ const struct config_enum_entry io_method_options[] = { int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ @@ -732,6 +735,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -855,6 +873,168 @@ pgaio_io_wait_for_free(void) +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (my_aio->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&my_aio->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + my_aio->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + my_aio->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - aio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + my_aio->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (my_aio->num_staged_ios > 0) + { + elog(DEBUG2, "submitting while acquiring free bb"); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = my_aio->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &aio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case AHS_IDLE: + case AHS_HANDED_OUT: + continue; + case AHS_DEFINED: /* should have been submitted above */ + case AHS_PREPARED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case AHS_REAPED: + case AHS_IN_FLIGHT: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + PgAioHandleRef ior; + + ior.aio_index = ioh - aio_ctl->io_handles; + ior.generation_upper = (uint32) (ioh->generation >> 32); + ior.generation_lower = (uint32) ioh->generation; + + pgaio_io_ref_wait(&ior); + elog(DEBUG2, "waited for io:%d to reclaim BB", + pgaio_io_get_id(ioh)); + + if (slist_is_empty(&my_aio->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case AHS_COMPLETED_SHARED: + case AHS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -929,6 +1109,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } /* @@ -939,6 +1120,7 @@ void pgaio_at_error(void) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 23adc5308e5..417526f3823 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioIOVDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioIOVShmemSize()); sz = add_size(sz, AioIOVDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_impl->shmem_size) sz = add_size(sz, pgaio_impl->shmem_size()); @@ -148,7 +221,10 @@ AioShmemInit(void) bool found; uint32 io_handle_off = 0; uint32 iovec_off = 0; + uint32 bounce_buffers_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_combine_limit; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; aio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) aio_ctl->io_handle_count = AioProcs() * io_max_concurrency; aio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + aio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; aio_ctl->backend_state = (PgAioPerBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -170,6 +247,35 @@ AioShmemInit(void) aio_ctl->iovecs = ShmemInitStruct("AioIOV", AioIOVShmemSize(), &found); aio_ctl->iovecs_data = ShmemInitStruct("AioIOVData", AioIOVDataShmemSize(), &found); + aio_ctl->bounce_buffers = ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), &found); + + bounce_buffers_data = ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), &found); + bounce_buffers_data = (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + aio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < aio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &aio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->subject = ASI_INVALID; + ioh->state = AHS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < aio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioPerBackend *bs = &aio_ctl->backend_state[procno]; @@ -177,9 +283,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -201,6 +311,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b2999b86c24..39e91ebd2a5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3233,6 +3233,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5893eb29228..da6e248a29e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -848,6 +848,8 @@ #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 5cf14472ebd..d1932b7393c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a5b12b48f99..dc52d6165d4 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2104,6 +2104,7 @@ Permutation PermutationStep PermutationStepBlocker PermutationStepBlockerType +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleFlags -- 2.45.2.746.g06e570c0df.dirty --oy2jwuii6tssbict Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0015-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2 14/20] aio: Add bounce buffers @ 2024-11-25 21:35 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2024-11-25 21:35 UTC (permalink / raw) --- src/include/storage/aio.h | 18 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 182 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 118 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 419 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index ff44dac5bb2..1bef475b0a9 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -222,6 +222,9 @@ typedef struct PgAioHandleSharedCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + /* * How many callbacks can be registered for one IO handle. Currently we only * need two, but it's not hard to imagine needing a few more. @@ -294,6 +297,20 @@ extern void pgaio_result_log(PgAioResult result, const PgAioSubjectData *subject +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -354,6 +371,7 @@ typedef enum IoMethod extern const struct config_enum_entry io_method_options[]; extern int io_method; extern int io_max_concurrency; +extern int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index d2dc1516bdf..2065bde79c3 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -91,6 +91,12 @@ struct PgAioHandle /* index into PgAioCtl->iovecs */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -130,11 +136,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioPerBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -162,6 +180,12 @@ typedef struct PgAioPerBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioPerBackend; @@ -187,6 +211,15 @@ typedef struct PgAioCtl */ uint64 *iovecs_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 2d55720a54c..0cdd0c13ffb 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 893f4ffe428..0076ea4aa10 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -395,6 +395,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 2439ce3740d..e829e1752ca 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -54,6 +54,8 @@ static void pgaio_io_resowner_register(PgAioHandle *ioh); static void pgaio_io_wait_for_free(void); static PgAioHandle *pgaio_io_from_ref(PgAioHandleRef *ior, uint64 *ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ @@ -68,6 +70,7 @@ const struct config_enum_entry io_method_options[] = { int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ @@ -732,6 +735,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -855,6 +873,168 @@ pgaio_io_wait_for_free(void) +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (my_aio->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&my_aio->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + my_aio->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + my_aio->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - aio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + my_aio->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (my_aio->num_staged_ios > 0) + { + elog(DEBUG2, "submitting while acquiring free bb"); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = my_aio->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &aio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case AHS_IDLE: + case AHS_HANDED_OUT: + continue; + case AHS_DEFINED: /* should have been submitted above */ + case AHS_PREPARED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case AHS_REAPED: + case AHS_IN_FLIGHT: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + PgAioHandleRef ior; + + ior.aio_index = ioh - aio_ctl->io_handles; + ior.generation_upper = (uint32) (ioh->generation >> 32); + ior.generation_lower = (uint32) ioh->generation; + + pgaio_io_ref_wait(&ior); + elog(DEBUG2, "waited for io:%d to reclaim BB", + pgaio_io_get_id(ioh)); + + if (slist_is_empty(&my_aio->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case AHS_COMPLETED_SHARED: + case AHS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -929,6 +1109,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } /* @@ -939,6 +1120,7 @@ void pgaio_at_error(void) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 23adc5308e5..417526f3823 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioIOVDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioIOVShmemSize()); sz = add_size(sz, AioIOVDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_impl->shmem_size) sz = add_size(sz, pgaio_impl->shmem_size()); @@ -148,7 +221,10 @@ AioShmemInit(void) bool found; uint32 io_handle_off = 0; uint32 iovec_off = 0; + uint32 bounce_buffers_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_combine_limit; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; aio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) aio_ctl->io_handle_count = AioProcs() * io_max_concurrency; aio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + aio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; aio_ctl->backend_state = (PgAioPerBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -170,6 +247,35 @@ AioShmemInit(void) aio_ctl->iovecs = ShmemInitStruct("AioIOV", AioIOVShmemSize(), &found); aio_ctl->iovecs_data = ShmemInitStruct("AioIOVData", AioIOVDataShmemSize(), &found); + aio_ctl->bounce_buffers = ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), &found); + + bounce_buffers_data = ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), &found); + bounce_buffers_data = (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + aio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < aio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &aio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->subject = ASI_INVALID; + ioh->state = AHS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < aio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioPerBackend *bs = &aio_ctl->backend_state[procno]; @@ -177,9 +283,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -201,6 +311,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b2999b86c24..39e91ebd2a5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3233,6 +3233,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5893eb29228..da6e248a29e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -848,6 +848,8 @@ #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 5cf14472ebd..d1932b7393c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a5b12b48f99..dc52d6165d4 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2104,6 +2104,7 @@ Permutation PermutationStep PermutationStepBlocker PermutationStepBlockerType +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleFlags -- 2.45.2.746.g06e570c0df.dirty --oy2jwuii6tssbict Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0015-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2 14/20] aio: Add bounce buffers @ 2024-11-25 21:35 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2024-11-25 21:35 UTC (permalink / raw) --- src/include/storage/aio.h | 18 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 182 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 118 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 419 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index ff44dac5bb2..1bef475b0a9 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -222,6 +222,9 @@ typedef struct PgAioHandleSharedCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + /* * How many callbacks can be registered for one IO handle. Currently we only * need two, but it's not hard to imagine needing a few more. @@ -294,6 +297,20 @@ extern void pgaio_result_log(PgAioResult result, const PgAioSubjectData *subject +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -354,6 +371,7 @@ typedef enum IoMethod extern const struct config_enum_entry io_method_options[]; extern int io_method; extern int io_max_concurrency; +extern int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index d2dc1516bdf..2065bde79c3 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -91,6 +91,12 @@ struct PgAioHandle /* index into PgAioCtl->iovecs */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -130,11 +136,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioPerBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -162,6 +180,12 @@ typedef struct PgAioPerBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioPerBackend; @@ -187,6 +211,15 @@ typedef struct PgAioCtl */ uint64 *iovecs_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 2d55720a54c..0cdd0c13ffb 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 893f4ffe428..0076ea4aa10 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -395,6 +395,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 2439ce3740d..e829e1752ca 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -54,6 +54,8 @@ static void pgaio_io_resowner_register(PgAioHandle *ioh); static void pgaio_io_wait_for_free(void); static PgAioHandle *pgaio_io_from_ref(PgAioHandleRef *ior, uint64 *ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ @@ -68,6 +70,7 @@ const struct config_enum_entry io_method_options[] = { int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ @@ -732,6 +735,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -855,6 +873,168 @@ pgaio_io_wait_for_free(void) +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (my_aio->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&my_aio->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + my_aio->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + my_aio->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - aio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + my_aio->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (my_aio->num_staged_ios > 0) + { + elog(DEBUG2, "submitting while acquiring free bb"); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = my_aio->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &aio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case AHS_IDLE: + case AHS_HANDED_OUT: + continue; + case AHS_DEFINED: /* should have been submitted above */ + case AHS_PREPARED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case AHS_REAPED: + case AHS_IN_FLIGHT: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + PgAioHandleRef ior; + + ior.aio_index = ioh - aio_ctl->io_handles; + ior.generation_upper = (uint32) (ioh->generation >> 32); + ior.generation_lower = (uint32) ioh->generation; + + pgaio_io_ref_wait(&ior); + elog(DEBUG2, "waited for io:%d to reclaim BB", + pgaio_io_get_id(ioh)); + + if (slist_is_empty(&my_aio->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case AHS_COMPLETED_SHARED: + case AHS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -929,6 +1109,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } /* @@ -939,6 +1120,7 @@ void pgaio_at_error(void) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 23adc5308e5..417526f3823 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioIOVDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioIOVShmemSize()); sz = add_size(sz, AioIOVDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_impl->shmem_size) sz = add_size(sz, pgaio_impl->shmem_size()); @@ -148,7 +221,10 @@ AioShmemInit(void) bool found; uint32 io_handle_off = 0; uint32 iovec_off = 0; + uint32 bounce_buffers_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_combine_limit; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; aio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) aio_ctl->io_handle_count = AioProcs() * io_max_concurrency; aio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + aio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; aio_ctl->backend_state = (PgAioPerBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -170,6 +247,35 @@ AioShmemInit(void) aio_ctl->iovecs = ShmemInitStruct("AioIOV", AioIOVShmemSize(), &found); aio_ctl->iovecs_data = ShmemInitStruct("AioIOVData", AioIOVDataShmemSize(), &found); + aio_ctl->bounce_buffers = ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), &found); + + bounce_buffers_data = ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), &found); + bounce_buffers_data = (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + aio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < aio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &aio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->subject = ASI_INVALID; + ioh->state = AHS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < aio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioPerBackend *bs = &aio_ctl->backend_state[procno]; @@ -177,9 +283,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -201,6 +311,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b2999b86c24..39e91ebd2a5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3233,6 +3233,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5893eb29228..da6e248a29e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -848,6 +848,8 @@ #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 5cf14472ebd..d1932b7393c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a5b12b48f99..dc52d6165d4 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2104,6 +2104,7 @@ Permutation PermutationStep PermutationStepBlocker PermutationStepBlockerType +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleFlags -- 2.45.2.746.g06e570c0df.dirty --oy2jwuii6tssbict Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0015-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.3 23/30] aio: Add bounce buffers @ 2025-01-22 22:13 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-01-22 22:13 UTC (permalink / raw) --- src/include/storage/aio.h | 19 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 180 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 423 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 6f36a0b9e4d..30b08495f3d 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -247,6 +247,10 @@ typedef struct PgAioHandleCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + + /* AIO API */ @@ -330,6 +334,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -345,6 +363,7 @@ extern void assign_io_method(int newval, void *extra); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index eff544ce621..531532e306a 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -97,6 +97,12 @@ struct PgAioHandle */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -133,11 +139,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -165,6 +183,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -190,6 +214,15 @@ typedef struct PgAioCtl */ uint64 *handle_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 1b6f9d2c40b..dacff46ad12 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -412,6 +412,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index b3b4e74c3ce..431f2c2e5af 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -55,6 +55,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -69,6 +71,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -588,6 +591,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -874,6 +892,166 @@ pgaio_have_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -904,6 +1082,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!pgaio_my_backend->handed_out_io); + Assert(!pgaio_my_backend->handed_out_bb); } /* @@ -914,6 +1093,7 @@ void pgaio_at_error(void) { Assert(!pgaio_my_backend->handed_out_io); + Assert(!pgaio_my_backend->handed_out_bb); } void diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 76fcdf64670..a4f4a0b698e 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8a83dcc820d..57865d45124 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3234,6 +3234,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5005e65cee0..294d661ebf4 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -853,6 +853,8 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index e5d852b5ee6..9db3c07326c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index be2dd22f1d7..b3f06711e6a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2110,6 +2110,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --gbq4ah2rmhae7qhd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.3-0024-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.3 23/30] aio: Add bounce buffers @ 2025-01-22 22:13 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-01-22 22:13 UTC (permalink / raw) --- src/include/storage/aio.h | 19 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 180 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 423 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 6f36a0b9e4d..30b08495f3d 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -247,6 +247,10 @@ typedef struct PgAioHandleCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + + /* AIO API */ @@ -330,6 +334,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -345,6 +363,7 @@ extern void assign_io_method(int newval, void *extra); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index eff544ce621..531532e306a 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -97,6 +97,12 @@ struct PgAioHandle */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -133,11 +139,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -165,6 +183,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -190,6 +214,15 @@ typedef struct PgAioCtl */ uint64 *handle_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 1b6f9d2c40b..dacff46ad12 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -412,6 +412,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index b3b4e74c3ce..431f2c2e5af 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -55,6 +55,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -69,6 +71,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -588,6 +591,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -874,6 +892,166 @@ pgaio_have_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -904,6 +1082,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!pgaio_my_backend->handed_out_io); + Assert(!pgaio_my_backend->handed_out_bb); } /* @@ -914,6 +1093,7 @@ void pgaio_at_error(void) { Assert(!pgaio_my_backend->handed_out_io); + Assert(!pgaio_my_backend->handed_out_bb); } void diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 76fcdf64670..a4f4a0b698e 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 8a83dcc820d..57865d45124 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3234,6 +3234,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5005e65cee0..294d661ebf4 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -853,6 +853,8 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index e5d852b5ee6..9db3c07326c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index be2dd22f1d7..b3f06711e6a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2110,6 +2110,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --gbq4ah2rmhae7qhd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.3-0024-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.4 23/29] aio: Add bounce buffers @ 2025-02-18 01:03 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-02-18 01:03 UTC (permalink / raw) --- src/include/storage/aio.h | 19 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 497 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 6b34422607c..2a50683adc5 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -247,6 +247,10 @@ typedef struct PgAioHandleCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + + /* AIO API */ @@ -332,6 +336,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -344,6 +362,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index ac26aff80b6..9c82e01ed17 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -99,6 +99,12 @@ struct PgAioHandle */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -135,11 +141,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -170,6 +188,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -195,6 +219,15 @@ typedef struct PgAioCtl */ uint64 *handle_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 55e64194ded..191fb21e6a7 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -404,6 +404,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index f2d763180d1..fc82908c338 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -61,6 +61,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -75,6 +77,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -642,6 +645,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1013,6 +1031,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 87eac5e961c..c6b29a7b134 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 15954f42d4e..58120b7add2 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3266,6 +3266,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 50fde0ba2c3..2214846a0b1 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -206,6 +206,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 76b9cec1e26..de00346b549 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index e7c7c6a6db6..822211f5dd4 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -63,6 +63,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 15851565853..15a548cff1a 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -427,6 +428,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index d4734b85c0d..d216785c3c8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2113,6 +2113,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --4ckqsto27zwk2eqr Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.4-0024-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.4 23/29] aio: Add bounce buffers @ 2025-02-18 01:03 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-02-18 01:03 UTC (permalink / raw) --- src/include/storage/aio.h | 19 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 497 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 6b34422607c..2a50683adc5 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -247,6 +247,10 @@ typedef struct PgAioHandleCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + + /* AIO API */ @@ -332,6 +336,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -344,6 +362,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index ac26aff80b6..9c82e01ed17 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -99,6 +99,12 @@ struct PgAioHandle */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -135,11 +141,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -170,6 +188,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -195,6 +219,15 @@ typedef struct PgAioCtl */ uint64 *handle_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 55e64194ded..191fb21e6a7 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -404,6 +404,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index f2d763180d1..fc82908c338 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -61,6 +61,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -75,6 +77,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -642,6 +645,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1013,6 +1031,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 87eac5e961c..c6b29a7b134 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 15954f42d4e..58120b7add2 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3266,6 +3266,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 50fde0ba2c3..2214846a0b1 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -206,6 +206,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 76b9cec1e26..de00346b549 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index e7c7c6a6db6..822211f5dd4 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -63,6 +63,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 15851565853..15a548cff1a 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -427,6 +428,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index d4734b85c0d..d216785c3c8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2113,6 +2113,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --4ckqsto27zwk2eqr Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.4-0024-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.5 23/30] aio: Add bounce buffers @ 2025-02-18 01:03 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-02-18 01:03 UTC (permalink / raw) --- src/include/storage/aio.h | 19 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 497 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 4913ff723a9..ce5c18424bd 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -247,6 +247,10 @@ typedef struct PgAioHandleCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + + /* AIO API */ @@ -333,6 +337,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -345,6 +363,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index f905b13e85f..835dafb5439 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -102,6 +102,12 @@ struct PgAioHandle */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -138,11 +144,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -173,6 +191,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -198,6 +222,15 @@ typedef struct PgAioCtl */ uint64 *handle_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index de6c7db894b..bd9436d665d 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -404,6 +404,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 3af61227ea8..ad0e033390c 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -61,6 +61,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -75,6 +77,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -638,6 +641,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1009,6 +1027,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 2205658cd9a..a56126a772a 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4317cfc9d2f..41a4025c66d 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3268,6 +3268,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 8738ad51bf1..291ca8ab38d 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 76b9cec1e26..de00346b549 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b35cf61cd36..86b46e93536 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2128,6 +2128,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --abvteypvk35ocehs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.5-0024-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.6 27/34] aio: Add bounce buffers @ 2025-03-07 20:47 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-07 20:47 UTC (permalink / raw) --- src/include/storage/aio.h | 18 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 12 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 3ce8763ebe8..cdf54b90b15 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -249,6 +249,9 @@ struct PgAioHandleCallbacks #define PGAIO_HANDLE_MAX_CALLBACKS 4 +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + /* -------------------------------------------------------------------------------- * IO Handles @@ -333,6 +336,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -345,6 +362,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 2412c5e7ecb..9471f2a92be 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -94,6 +94,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -149,11 +155,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -184,6 +202,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -211,6 +235,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index de6c7db894b..bd9436d665d 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -404,6 +404,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index c2b34a95e1c..e1b39b5d44c 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -663,6 +666,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1045,6 +1063,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 2205658cd9a..a56126a772a 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -149,6 +222,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -172,6 +249,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -179,9 +290,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -203,6 +318,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4317cfc9d2f..41a4025c66d 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3268,6 +3268,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 8738ad51bf1..291ca8ab38d 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 76b9cec1e26..de00346b549 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e5a13e6e6dd..fc379ff27eb 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2129,6 +2129,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --6g5xner6ro2tsnwz Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.6-0028-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.8 30/38] aio: Add bounce buffers @ 2025-03-07 20:47 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-07 20:47 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index a4d9b51acfe..3047dc77ce1 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -337,6 +337,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -349,6 +363,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 98ae66921f5..166a66f7062 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index b7dfb80b4b2..bb0bfc71ea6 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 3f126762a22..8ad6be06ca1 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -664,6 +667,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1045,6 +1063,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index d0d916ea5c9..c32d8e6cb3d 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -131,11 +184,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -150,6 +223,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -161,6 +237,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -173,6 +250,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -180,9 +291,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -204,6 +319,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4cc19bef686..7290793c9a1 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3268,6 +3268,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 7e8c3dcb175..ea71e9075ff 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1748befca16..6ffd4e82976 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --ow5flh3n247znjrs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.8-0031-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.7 28/35] aio: Add bounce buffers @ 2025-03-07 20:47 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-07 20:47 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index a4d9b51acfe..3047dc77ce1 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -337,6 +337,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -349,6 +363,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 2580e34e43b..ca6cbde11a6 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index b7dfb80b4b2..bb0bfc71ea6 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index f31e52f7fe0..a9c351eb0dc 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -663,6 +666,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index d0d916ea5c9..c32d8e6cb3d 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -131,11 +184,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -150,6 +223,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -161,6 +237,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -173,6 +250,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -180,9 +291,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -204,6 +319,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index d4e63c75cb6..3a375b92652 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3259,6 +3259,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index c9e9e850a99..24e1af803b0 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 7b9affbeb80..596b9a56265 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --vsphh7g5lukufvxs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.7-0029-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.8 30/38] aio: Add bounce buffers @ 2025-03-07 20:47 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-07 20:47 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index a4d9b51acfe..3047dc77ce1 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -337,6 +337,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -349,6 +363,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 98ae66921f5..166a66f7062 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index b7dfb80b4b2..bb0bfc71ea6 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 3f126762a22..8ad6be06ca1 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -664,6 +667,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1045,6 +1063,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index d0d916ea5c9..c32d8e6cb3d 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -131,11 +184,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_method_ops->shmem_size) sz = add_size(sz, pgaio_method_ops->shmem_size()); @@ -150,6 +223,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -161,6 +237,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -173,6 +250,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -180,9 +291,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -204,6 +319,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4cc19bef686..7290793c9a1 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3268,6 +3268,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 7e8c3dcb175..ea71e9075ff 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1748befca16..6ffd4e82976 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --ow5flh3n247znjrs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.8-0031-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.9 22/30] aio: Add bounce buffers @ 2025-03-15 16:29 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-15 16:29 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index c0db8f9754f..c673627a358 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -344,6 +344,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -356,6 +370,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 98ae66921f5..166a66f7062 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index b7dfb80b4b2..bb0bfc71ea6 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index a0504ee84ee..46d83da60f7 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 4e405ce7ca8..b9732d989ff 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -93,6 +93,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -118,6 +144,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -141,11 +194,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -161,6 +234,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -172,6 +248,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -184,6 +261,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -191,9 +302,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -215,6 +330,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4cc19bef686..7290793c9a1 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3268,6 +3268,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 7e8c3dcb175..ea71e9075ff 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1748befca16..6ffd4e82976 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --23jbdfobqrqxnmx5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.9-0023-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.13 20/28] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index b182c0c78ca..f9a6ac8c625 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -349,6 +349,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -361,6 +375,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 7f18da2c856..833f97361a1 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -127,6 +127,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -182,11 +188,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -217,6 +236,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -244,6 +269,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index 9db5d776b61..134986d3af1 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -119,4 +119,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index ddd59404a59..534aaad22bc 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -406,6 +406,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits (see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer can be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 86f7250b7a5..cff48964d07 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) if (ioh->state != PGAIO_HS_HANDED_OUT) dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node); + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1046,6 +1064,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5d729102f46..e885a682258 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3303,6 +3303,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ac6ae0a9489..4a5d04ef419 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index c668d516f61..f93a698ae34 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -86,6 +86,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index c7e857800dc..0cbb71fecfa 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -51,6 +51,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -617,6 +618,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 9442a4841aa..49e1faecacf 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2133,6 +2133,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --pro7bqageygxfsvg Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.13-0021-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.14 21/29] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index bfe0d93683b..f91f0afc5a5 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -353,6 +353,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -365,6 +379,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 7f18da2c856..833f97361a1 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -127,6 +127,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -182,11 +188,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -217,6 +236,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -244,6 +269,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index 156132cde03..b23ed0ec47d 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -134,4 +134,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index ddd59404a59..534aaad22bc 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -406,6 +406,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits (see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer can be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 86f7250b7a5..cff48964d07 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) if (ioh->state != PGAIO_HS_HANDED_OUT) dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node); + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1046,6 +1064,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4eaeca89f2c..bd49b302293 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3293,6 +3293,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ff56a1f0732..2c6456e907f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index e495481c41e..e2a81235166 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -87,6 +87,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 35c6b331923..5d8f255e449 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -52,6 +52,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -671,6 +672,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 1279b69422a..74ba36eb78c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2134,6 +2134,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --m2na7lgr3zfazgom Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.14-0022-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.15 10/18] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index bfe0d93683b..f91f0afc5a5 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -353,6 +353,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -365,6 +379,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 7f18da2c856..833f97361a1 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -127,6 +127,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -182,11 +188,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -217,6 +236,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -244,6 +269,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index 18183366077..3c18dade49c 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -134,4 +134,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index ddd59404a59..534aaad22bc 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -406,6 +406,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits (see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer can be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 86f7250b7a5..cff48964d07 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) if (ioh->state != PGAIO_HS_HANDED_OUT) dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node); + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1046,6 +1064,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 4eaeca89f2c..bd49b302293 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3293,6 +3293,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ff56a1f0732..2c6456e907f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index e495481c41e..e2a81235166 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -87,6 +87,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index bef3b0e3ad0..2a7f4378ef3 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -52,6 +52,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -669,6 +670,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index b66cecd8799..3a67ee01b46 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2137,6 +2137,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --xevce4sdbnyxplun Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.15-0011-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.11 20/27] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 9a0868a270c..09d6d9fe1bc 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -349,6 +349,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -361,6 +375,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 6e6033519b2..447208c8f12 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index debe8163d4e..02950ee56a7 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 012cf3f305a..57e6f2fef07 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 3c3383b1318..ef344c32c61 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2e17dbc8c9e..89f9b1fbab7 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3303,6 +3303,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d5f5f3dbda4..762194c61ad 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index fc55b985dad..1e2127fe888 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -526,6 +527,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 53abd54edd8..d633c732162 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --bjnmbpad43bpmfxt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.11-0021-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.10 19/28] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index c0db8f9754f..c673627a358 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -344,6 +344,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -356,6 +370,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index da4d9c341f0..4378d51908f 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 6df4826163e..628018ff246 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index a0504ee84ee..46d83da60f7 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 4e405ce7ca8..b9732d989ff 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -93,6 +93,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -118,6 +144,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -141,11 +194,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -161,6 +234,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -172,6 +248,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -184,6 +261,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -191,9 +302,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -215,6 +330,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index ea0d9dfb841..960ced7c274 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3278,6 +3278,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 0748601ec20..799851cc347 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 53abd54edd8..d633c732162 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --w6dfit2y42fwvotd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.10-0020-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.13 20/28] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index b182c0c78ca..f9a6ac8c625 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -349,6 +349,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -361,6 +375,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 7f18da2c856..833f97361a1 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -127,6 +127,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -182,11 +188,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -217,6 +236,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -244,6 +269,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index 9db5d776b61..134986d3af1 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -119,4 +119,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index ddd59404a59..534aaad22bc 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -406,6 +406,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits (see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer can be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 86f7250b7a5..cff48964d07 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) if (ioh->state != PGAIO_HS_HANDED_OUT) dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node); + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1046,6 +1064,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5d729102f46..e885a682258 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3303,6 +3303,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index ac6ae0a9489..4a5d04ef419 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index c668d516f61..f93a698ae34 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -86,6 +86,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index c7e857800dc..0cbb71fecfa 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -51,6 +51,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -617,6 +618,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 9442a4841aa..49e1faecacf 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2133,6 +2133,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --pro7bqageygxfsvg Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.13-0021-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.10 19/28] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index c0db8f9754f..c673627a358 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -344,6 +344,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -356,6 +370,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index da4d9c341f0..4378d51908f 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index b0b9a2a5c97..09968e17855 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 6df4826163e..628018ff246 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index a0504ee84ee..46d83da60f7 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 4e405ce7ca8..b9732d989ff 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -93,6 +93,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -118,6 +144,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -141,11 +194,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -161,6 +234,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * PG_IOV_MAX; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -172,6 +248,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -184,6 +261,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -191,9 +302,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -215,6 +330,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += PG_IOV_MAX; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index ea0d9dfb841..960ced7c274 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3278,6 +3278,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 0748601ec20..799851cc347 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -209,6 +209,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 81b4d732206..60c01b251fe 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -525,6 +526,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 53abd54edd8..d633c732162 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --w6dfit2y42fwvotd Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.10-0020-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.11 20/27] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 9a0868a270c..09d6d9fe1bc 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -349,6 +349,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -361,6 +375,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 6e6033519b2..447208c8f12 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index debe8163d4e..02950ee56a7 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 012cf3f305a..57e6f2fef07 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 3c3383b1318..ef344c32c61 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 2e17dbc8c9e..89f9b1fbab7 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3303,6 +3303,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d5f5f3dbda4..762194c61ad 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 3d4e9bb070f..16f248619fc 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -76,6 +76,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index fc55b985dad..1e2127fe888 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -526,6 +527,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 53abd54edd8..d633c732162 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2132,6 +2132,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --bjnmbpad43bpmfxt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.11-0021-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v2.12 20/28] aio: Add bounce buffers @ 2025-03-18 18:40 Andres Freund <andres@anarazel.de> 0 siblings, 0 replies; 23+ messages in thread From: Andres Freund @ 2025-03-18 18:40 UTC (permalink / raw) --- src/include/storage/aio.h | 15 ++ src/include/storage/aio_internal.h | 34 ++++ src/include/storage/aio_types.h | 2 + src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 178 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 123 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/test/modules/test_aio/test_aio--1.0.sql | 21 +++ src/test/modules/test_aio/test_aio.c | 55 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 496 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index 9a0868a270c..09d6d9fe1bc 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -349,6 +349,20 @@ extern bool pgaio_have_staged(void); +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(struct dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- @@ -361,6 +375,7 @@ extern void pgaio_closing_fd(int fd); /* GUCs */ extern PGDLLIMPORT int io_method; extern PGDLLIMPORT int io_max_concurrency; +extern PGDLLIMPORT int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index 0dff909e4ad..776c6985220 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -122,6 +122,12 @@ struct PgAioHandle /* raw result of the IO operation */ int32 result; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -177,11 +183,24 @@ struct PgAioHandle }; +/* typedef is in aio_types.h */ +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -212,6 +231,12 @@ typedef struct PgAioBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioBackend; @@ -239,6 +264,15 @@ typedef struct PgAioCtl uint32 io_handle_count; PgAioHandle *io_handles; + + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint32 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; } PgAioCtl; diff --git a/src/include/storage/aio_types.h b/src/include/storage/aio_types.h index debe8163d4e..02950ee56a7 100644 --- a/src/include/storage/aio_types.h +++ b/src/include/storage/aio_types.h @@ -118,4 +118,6 @@ typedef struct PgAioReturn } PgAioReturn; +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + #endif /* AIO_TYPES_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index aede4bfc820..7e2ec224169 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 24eec5776cd..b7d43ba8422 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -405,6 +405,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 0d52f72dfdc..f50ae7cdc23 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -62,6 +62,8 @@ static PgAioHandle *pgaio_io_from_wref(PgAioWaitRef *iow, uint64 *ref_generation static const char *pgaio_io_state_get_name(PgAioHandleState s); static void pgaio_io_wait(PgAioHandle *ioh, uint64 ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ const struct config_enum_entry io_method_options[] = { @@ -76,6 +78,7 @@ const struct config_enum_entry io_method_options[] = { /* GUCs */ int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ PgAioCtl *pgaio_ctl; @@ -662,6 +665,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) if (ioh->state != PGAIO_HS_HANDED_OUT) dclist_delete_from(&pgaio_my_backend->in_flight_ios, &ioh->node); + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -1044,6 +1062,166 @@ pgaio_submit_staged(void) +/* -------------------------------------------------------------------------------- + * Functions primarily related to PgAioBounceBuffer + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (pgaio_my_backend->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * XXX: It probably is not a good idea to have bounce buffers be per + * backend, that's a fair bit of memory. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&pgaio_my_backend->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + pgaio_my_backend->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + pgaio_my_backend->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - pgaio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (pgaio_my_backend->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&pgaio_my_backend->idle_bbs, &bb->node); + pgaio_my_backend->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (pgaio_my_backend->num_staged_ios > 0) + { + pgaio_debug(DEBUG2, "submitting %d, while acquiring free bb", + pgaio_my_backend->num_staged_ios); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = pgaio_my_backend->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &pgaio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case PGAIO_HS_IDLE: + case PGAIO_HS_HANDED_OUT: + continue; + case PGAIO_HS_DEFINED: /* should have been submitted above */ + case PGAIO_HS_STAGED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case PGAIO_HS_COMPLETED_IO: + case PGAIO_HS_SUBMITTED: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + pgaio_debug_io(DEBUG2, ioh, + "waiting for IO to reclaim BB with %d in flight", + dclist_count(&pgaio_my_backend->in_flight_ios)); + + /* see comment in pgaio_io_wait_for_free() about raciness */ + pgaio_io_wait(ioh, ioh->generation); + + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case PGAIO_HS_COMPLETED_SHARED: + case PGAIO_HS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&pgaio_my_backend->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&pgaio_my_backend->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Other * -------------------------------------------------------------------------------- diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 885c3940c66..95b10933fed 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -88,6 +88,32 @@ AioHandleDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -113,6 +139,33 @@ AioChooseMaxConcurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -136,11 +189,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioHandleIOVShmemSize()); sz = add_size(sz, AioHandleDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); /* Reserve space for method specific resources. */ if (pgaio_method_ops->shmem_size) @@ -156,6 +229,9 @@ AioShmemInit(void) uint32 io_handle_off = 0; uint32 iovec_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_max_combine_limit; + uint32 bounce_buffers_off = 0; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; pgaio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -167,6 +243,7 @@ AioShmemInit(void) pgaio_ctl->io_handle_count = AioProcs() * io_max_concurrency; pgaio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + pgaio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; pgaio_ctl->backend_state = (PgAioBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -179,6 +256,40 @@ AioShmemInit(void) pgaio_ctl->handle_data = (uint64 *) ShmemInitStruct("AioHandleData", AioHandleDataShmemSize(), &found); + pgaio_ctl->bounce_buffers = (PgAioBounceBuffer *) + ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), + &found); + + bounce_buffers_data = + ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), + &found); + bounce_buffers_data = + (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + pgaio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < pgaio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &pgaio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->target = PGAIO_TID_INVALID; + ioh->state = PGAIO_HS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < pgaio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioBackend *bs = &pgaio_ctl->backend_state[procno]; @@ -186,9 +297,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -210,6 +325,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_max_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &pgaio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 5d729102f46..e885a682258 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3303,6 +3303,19 @@ struct config_int ConfigureNamesInt[] = check_io_max_concurrency, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_IO, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 771fe4cbe35..ef9585ac991 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -211,6 +211,8 @@ # -1 sets based on shared_buffers # (change requires restart) #io_workers = 3 # 1-32; +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) # - Worker Processes - diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index d39f3e1b655..81e7e27965a 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResourceElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResourceElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -742,6 +744,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1111,3 +1120,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/test/modules/test_aio/test_aio--1.0.sql b/src/test/modules/test_aio/test_aio--1.0.sql index 602615e39be..3187bd859b7 100644 --- a/src/test/modules/test_aio/test_aio--1.0.sql +++ b/src/test/modules/test_aio/test_aio--1.0.sql @@ -80,6 +80,27 @@ RETURNS pg_catalog.void STRICT AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION bb_get_and_error() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_twice() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_get_release() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION bb_release_last() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; + + /* * Injection point related functions diff --git a/src/test/modules/test_aio/test_aio.c b/src/test/modules/test_aio/test_aio.c index 0f34892a17e..2e12e60d3fb 100644 --- a/src/test/modules/test_aio/test_aio.c +++ b/src/test/modules/test_aio/test_aio.c @@ -53,6 +53,7 @@ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; static PgAioHandle *last_handle; +static PgAioBounceBuffer *last_bb; @@ -581,6 +582,60 @@ batch_end(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +PG_FUNCTION_INFO_V1(bb_get); +Datum +bb_get(PG_FUNCTION_ARGS) +{ + last_bb = pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_release_last); +Datum +bb_release_last(PG_FUNCTION_ARGS) +{ + if (!last_bb) + elog(ERROR, "no bb"); + + pgaio_bounce_buffer_release(last_bb); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_and_error); +Datum +bb_get_and_error(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + + elog(ERROR, "as you command"); + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(bb_get_twice); +Datum +bb_get_twice(PG_FUNCTION_ARGS) +{ + pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_get(); + + PG_RETURN_VOID(); +} + + +PG_FUNCTION_INFO_V1(bb_get_release); +Datum +bb_get_release(PG_FUNCTION_ARGS) +{ + PgAioBounceBuffer *bb; + + bb = pgaio_bounce_buffer_get(); + pgaio_bounce_buffer_release(bb); + + PG_RETURN_VOID(); +} + #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void inj_io_short_read(const char *name, const void *private_data); extern PGDLLEXPORT void inj_io_reopen(const char *name, const void *private_data); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e253f4937a2..cabe406294a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2133,6 +2133,7 @@ PermutationStep PermutationStepBlocker PermutationStepBlockerType PgAioBackend +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleCallbackID -- 2.48.1.76.g4e746b1a31.dirty --5i73spx2p4vwf7fe Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2.12-0021-bufmgr-Implement-AIO-write-support.patch" ^ permalink raw reply [nested|flat] 23+ messages in thread
* [PATCH v3 1/1] remove toast reloptions @ 2026-04-07 20:30 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 23+ messages in thread From: Nathan Bossart @ 2026-04-07 20:30 UTC (permalink / raw) --- doc/src/sgml/ref/create_table.sgml | 51 ++++++------- src/backend/access/common/reloptions.c | 49 +++++------- src/backend/catalog/toasting.c | 31 ++++---- src/backend/commands/createas.c | 13 +--- src/backend/commands/repack.c | 15 +--- src/backend/commands/tablecmds.c | 19 +---- src/backend/commands/vacuum.c | 14 +--- src/backend/postmaster/autovacuum.c | 48 +++++------- src/backend/tcop/utility.c | 19 +---- src/bin/pg_upgrade/check.c | 75 +++++++++++++++++++ src/include/access/reloptions.h | 24 +++--- src/include/catalog/toasting.h | 6 +- .../injection_points/expected/vacuum.out | 24 +++--- .../modules/injection_points/sql/vacuum.sql | 20 ++--- src/test/regress/expected/alter_table.out | 9 --- src/test/regress/expected/reloptions.out | 56 ++------------ src/test/regress/expected/vacuum.out | 8 -- src/test/regress/sql/alter_table.sql | 4 - src/test/regress/sql/reloptions.sql | 32 +------- src/test/regress/sql/vacuum.sql | 8 -- 20 files changed, 199 insertions(+), 326 deletions(-) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index e342585c7f0..b8d2a657d10 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1589,14 +1589,10 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM Storage parameters for indexes are documented in <xref linkend="sql-createindex"/>. The storage parameters currently - available for tables are listed below. For many of these parameters, as - shown, there is an additional parameter with the same name prefixed with - <literal>toast.</literal>, which controls the behavior of the + available for tables are listed below. Unless otherwise noted, a table's + parameter value also controls the behavior of the table's secondary <acronym>TOAST</acronym> table, if any (see <xref linkend="storage-toast"/> for more information about TOAST). - If a table parameter value is set and the - equivalent <literal>toast.</literal> parameter is not, the TOAST table - will use the table's parameter value. Specifying these parameters for partitioned tables is not supported, but you may specify them for individual leaf partitions. </para> @@ -1622,7 +1618,8 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM updates</link> more likely. For a table whose entries are never updated, complete packing is the best choice, but in heavily updated tables smaller fillfactors are - appropriate. This parameter cannot be set for TOAST tables. + appropriate. This parameter does not affect the table's secondary TOAST + table. </para> </listitem> </varlistentry> @@ -1648,7 +1645,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM Note that the default setting is often close to optimal, and it is possible that setting this parameter could have negative effects in some cases. - This parameter cannot be set for TOAST tables. + This parameter does not affect the table's secondary TOAST table. </para> </listitem> </varlistentry> @@ -1671,7 +1668,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-enabled" xreflabel="autovacuum_enabled"> - <term><literal>autovacuum_enabled</literal>, <literal>toast.autovacuum_enabled</literal> (<type>boolean</type>) + <term><literal>autovacuum_enabled</literal> (<type>boolean</type>) <indexterm> <primary><varname>autovacuum_enabled</varname> storage parameter</primary> </indexterm> @@ -1696,7 +1693,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-vacuum-index-cleanup" xreflabel="vacuum_index_cleanup"> - <term><literal>vacuum_index_cleanup</literal>, <literal>toast.vacuum_index_cleanup</literal> (<type>enum</type>) + <term><literal>vacuum_index_cleanup</literal> (<type>enum</type>) <indexterm> <primary><varname>vacuum_index_cleanup</varname> storage parameter</primary> </indexterm> @@ -1722,7 +1719,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-vacuum-truncate" xreflabel="vacuum_truncate"> - <term><literal>vacuum_truncate</literal>, <literal>toast.vacuum_truncate</literal> (<type>boolean</type>) + <term><literal>vacuum_truncate</literal> (<type>boolean</type>) <indexterm> <primary><varname>vacuum_truncate</varname></primary> <secondary>storage parameter</secondary> @@ -1755,7 +1752,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-threshold" xreflabel="autovacuum_vacuum_threshold"> - <term><literal>autovacuum_vacuum_threshold</literal>, <literal>toast.autovacuum_vacuum_threshold</literal> (<type>integer</type>) + <term><literal>autovacuum_vacuum_threshold</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_vacuum_threshold</varname></primary> <secondary>storage parameter</secondary> @@ -1770,7 +1767,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-max-threshold" xreflabel="autovacuum_vacuum_max_threshold"> - <term><literal>autovacuum_vacuum_max_threshold</literal>, <literal>toast.autovacuum_vacuum_max_threshold</literal> (<type>integer</type>) + <term><literal>autovacuum_vacuum_max_threshold</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_vacuum_max_threshold</varname></primary> <secondary>storage parameter</secondary> @@ -1785,7 +1782,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-scale-factor" xreflabel="autovacuum_vacuum_scale_factor"> - <term><literal>autovacuum_vacuum_scale_factor</literal>, <literal>toast.autovacuum_vacuum_scale_factor</literal> (<type>floating point</type>) + <term><literal>autovacuum_vacuum_scale_factor</literal> (<type>floating point</type>) <indexterm> <primary><varname>autovacuum_vacuum_scale_factor</varname> </primary> <secondary>storage parameter</secondary> @@ -1800,7 +1797,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-insert-threshold" xreflabel="autovacuum_vacuum_insert_threshold"> - <term><literal>autovacuum_vacuum_insert_threshold</literal>, <literal>toast.autovacuum_vacuum_insert_threshold</literal> (<type>integer</type>) + <term><literal>autovacuum_vacuum_insert_threshold</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_vacuum_insert_threshold</varname></primary> <secondary>storage parameter</secondary> @@ -1815,7 +1812,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-insert-scale-factor" xreflabel="autovacuum_vacuum_insert_scale_factor"> - <term><literal>autovacuum_vacuum_insert_scale_factor</literal>, <literal>toast.autovacuum_vacuum_insert_scale_factor</literal> (<type>floating point</type>) + <term><literal>autovacuum_vacuum_insert_scale_factor</literal> (<type>floating point</type>) <indexterm> <primary><varname>autovacuum_vacuum_insert_scale_factor</varname> </primary> <secondary>storage parameter</secondary> @@ -1860,7 +1857,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-cost-delay" xreflabel="autovacuum_vacuum_cost_delay"> - <term><literal>autovacuum_vacuum_cost_delay</literal>, <literal>toast.autovacuum_vacuum_cost_delay</literal> (<type>floating point</type>) + <term><literal>autovacuum_vacuum_cost_delay</literal> (<type>floating point</type>) <indexterm> <primary><varname>autovacuum_vacuum_cost_delay</varname></primary> <secondary>storage parameter</secondary> @@ -1875,7 +1872,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-vacuum-cost-limit" xreflabel="autovacuum_vacuum_cost_limit"> - <term><literal>autovacuum_vacuum_cost_limit</literal>, <literal>toast.autovacuum_vacuum_cost_limit</literal> (<type>integer</type>) + <term><literal>autovacuum_vacuum_cost_limit</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_vacuum_cost_limit</varname></primary> <secondary>storage parameter</secondary> @@ -1890,7 +1887,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-freeze-min-age" xreflabel="autovacuum_freeze_min_age"> - <term><literal>autovacuum_freeze_min_age</literal>, <literal>toast.autovacuum_freeze_min_age</literal> (<type>integer</type>) + <term><literal>autovacuum_freeze_min_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_freeze_min_age</varname> storage parameter</primary> </indexterm> @@ -1907,7 +1904,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-freeze-max-age" xreflabel="autovacuum_freeze_max_age"> - <term><literal>autovacuum_freeze_max_age</literal>, <literal>toast.autovacuum_freeze_max_age</literal> (<type>integer</type>) + <term><literal>autovacuum_freeze_max_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_freeze_max_age</varname></primary> <secondary>storage parameter</secondary> @@ -1924,7 +1921,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-freeze-table-age" xreflabel="autovacuum_freeze_table_age"> - <term><literal>autovacuum_freeze_table_age</literal>, <literal>toast.autovacuum_freeze_table_age</literal> (<type>integer</type>) + <term><literal>autovacuum_freeze_table_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_freeze_table_age</varname> storage parameter</primary> </indexterm> @@ -1938,7 +1935,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-multixact-freeze-min-age" xreflabel="autovacuum_multixact_freeze_min_age"> - <term><literal>autovacuum_multixact_freeze_min_age</literal>, <literal>toast.autovacuum_multixact_freeze_min_age</literal> (<type>integer</type>) + <term><literal>autovacuum_multixact_freeze_min_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_multixact_freeze_min_age</varname> storage parameter</primary> </indexterm> @@ -1956,7 +1953,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-multixact-freeze-max-age" xreflabel="autovacuum_multixact_freeze_max_age"> - <term><literal>autovacuum_multixact_freeze_max_age</literal>, <literal>toast.autovacuum_multixact_freeze_max_age</literal> (<type>integer</type>) + <term><literal>autovacuum_multixact_freeze_max_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_multixact_freeze_max_age</varname></primary> <secondary>storage parameter</secondary> @@ -1975,7 +1972,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-autovacuum-multixact-freeze-table-age" xreflabel="autovacuum_multixact_freeze_table_age"> - <term><literal>autovacuum_multixact_freeze_table_age</literal>, <literal>toast.autovacuum_multixact_freeze_table_age</literal> (<type>integer</type>) + <term><literal>autovacuum_multixact_freeze_table_age</literal> (<type>integer</type>) <indexterm> <primary><varname>autovacuum_multixact_freeze_table_age</varname> storage parameter</primary> </indexterm> @@ -1989,7 +1986,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-log-autovacuum-min-duration" xreflabel="log_autovacuum_min_duration"> - <term><literal>log_autovacuum_min_duration</literal>, <literal>toast.log_autovacuum_min_duration</literal> (<type>integer</type>) + <term><literal>log_autovacuum_min_duration</literal> (<type>integer</type>) <indexterm> <primary><varname>log_autovacuum_min_duration</varname></primary> <secondary>storage parameter</secondary> @@ -2019,7 +2016,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM </varlistentry> <varlistentry id="reloption-vacuum-max-eager-freeze-failure-rate" xreflabel="vacuum_max_eager_freeze_failure_rate"> - <term><literal>vacuum_max_eager_freeze_failure_rate</literal>, <literal>toast.vacuum_max_eager_freeze_failure_rate</literal> (<type>floating point</type>) + <term><literal>vacuum_max_eager_freeze_failure_rate</literal> (<type>floating point</type>) <indexterm> <primary><varname>vacuum_max_eager_freeze_failure_rate</varname></primary> <secondary>storage parameter</secondary> @@ -2044,7 +2041,7 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM Declare the table as an additional catalog table for purposes of logical replication. See <xref linkend="logicaldecoding-capabilities"/> for details. - This parameter cannot be set for TOAST tables. + This parameter does not affect the table's secondary TOAST table. </para> </listitem> </varlistentry> diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 3e832c3797e..50d9805f3d8 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -111,7 +111,7 @@ static relopt_bool boolRelOpts[] = { "autovacuum_enabled", "Enables autovacuum in this relation", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, true @@ -172,7 +172,7 @@ static relopt_ternary ternaryRelOpts[] = { "vacuum_truncate", "Enables vacuum to truncate empty pages at the end of this table", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock } }, @@ -249,7 +249,7 @@ static relopt_int intRelOpts[] = { "autovacuum_vacuum_threshold", "Minimum number of tuple updates or deletes prior to vacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0, INT_MAX @@ -258,7 +258,7 @@ static relopt_int intRelOpts[] = { "autovacuum_vacuum_max_threshold", "Maximum number of tuple updates or deletes prior to vacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -2, -1, INT_MAX @@ -267,7 +267,7 @@ static relopt_int intRelOpts[] = { "autovacuum_vacuum_insert_threshold", "Minimum number of tuple inserts prior to vacuum, or -1 to disable insert vacuums", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -2, -1, INT_MAX @@ -285,7 +285,7 @@ static relopt_int intRelOpts[] = { "autovacuum_vacuum_cost_limit", "Vacuum cost amount available before napping, for autovacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 1, 10000 @@ -294,7 +294,7 @@ static relopt_int intRelOpts[] = { "autovacuum_freeze_min_age", "Minimum age at which VACUUM should freeze a table row, for autovacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0, 1000000000 @@ -303,7 +303,7 @@ static relopt_int intRelOpts[] = { "autovacuum_multixact_freeze_min_age", "Minimum multixact age at which VACUUM should freeze a row multixact's, for autovacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0, 1000000000 @@ -312,7 +312,7 @@ static relopt_int intRelOpts[] = { "autovacuum_freeze_max_age", "Age at which to autovacuum a table to prevent transaction ID wraparound", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 100000, 2000000000 @@ -321,7 +321,7 @@ static relopt_int intRelOpts[] = { "autovacuum_multixact_freeze_max_age", "Multixact age at which to autovacuum a table to prevent multixact wraparound", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 10000, 2000000000 @@ -330,7 +330,7 @@ static relopt_int intRelOpts[] = { "autovacuum_freeze_table_age", "Age at which VACUUM should perform a full table sweep to freeze row versions", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0, 2000000000 }, @@ -338,7 +338,7 @@ static relopt_int intRelOpts[] = { "autovacuum_multixact_freeze_table_age", "Age of multixact at which VACUUM should perform a full table sweep to freeze row versions", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0, 2000000000 }, @@ -346,7 +346,7 @@ static relopt_int intRelOpts[] = { "log_autovacuum_min_duration", "Sets the minimum execution time above which vacuum actions by autovacuum will be logged", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, -1, INT_MAX @@ -424,7 +424,7 @@ static relopt_real realRelOpts[] = { "autovacuum_vacuum_cost_delay", "Vacuum cost delay in milliseconds, for autovacuum", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0.0, 100.0 @@ -433,7 +433,7 @@ static relopt_real realRelOpts[] = { "autovacuum_vacuum_scale_factor", "Number of tuple updates or deletes prior to vacuum as a fraction of reltuples", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0.0, 100.0 @@ -442,7 +442,7 @@ static relopt_real realRelOpts[] = { "autovacuum_vacuum_insert_scale_factor", "Number of tuple inserts prior to vacuum as a fraction of reltuples", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0.0, 100.0 @@ -460,7 +460,7 @@ static relopt_real realRelOpts[] = { "vacuum_max_eager_freeze_failure_rate", "Fraction of pages in a relation vacuum can scan and fail to freeze before disabling eager scanning.", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, -1, 0.0, 1.0 @@ -554,7 +554,7 @@ static relopt_enum enumRelOpts[] = { "vacuum_index_cleanup", "Controls index vacuuming and index cleanup", - RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, StdRdOptIndexCleanupValues, @@ -2163,21 +2163,8 @@ view_reloptions(Datum reloptions, bool validate) bytea * heap_reloptions(char relkind, Datum reloptions, bool validate) { - StdRdOptions *rdopts; - switch (relkind) { - case RELKIND_TOASTVALUE: - rdopts = (StdRdOptions *) - default_reloptions(reloptions, validate, RELOPT_KIND_TOAST); - if (rdopts != NULL) - { - /* adjust default-only parameters for TOAST relations */ - rdopts->fillfactor = 100; - rdopts->autovacuum.analyze_threshold = -1; - rdopts->autovacuum.analyze_scale_factor = -1; - } - return (bytea *) rdopts; case RELKIND_RELATION: case RELKIND_MATVIEW: return default_reloptions(reloptions, validate, RELOPT_KIND_HEAP); diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 4aa52a4bd25..9f1f822702c 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -34,11 +34,11 @@ #include "utils/rel.h" #include "utils/syscache.h" -static void CheckAndCreateToastTable(Oid relOid, Datum reloptions, +static void CheckAndCreateToastTable(Oid relOid, LOCKMODE lockmode, bool check, Oid OIDOldToast); static bool create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, - Datum reloptions, LOCKMODE lockmode, bool check, + LOCKMODE lockmode, bool check, Oid OIDOldToast); static bool needs_toast_table(Relation rel); @@ -48,35 +48,30 @@ static bool needs_toast_table(Relation rel); * If the table needs a toast table, and doesn't already have one, * then create a toast table for it. * - * reloptions for the toast table can be passed, too. Pass (Datum) 0 - * for default reloptions. - * * We expect the caller to have verified that the relation is a table and have * already done any necessary permission checks. Callers expect this function * to end with CommandCounterIncrement if it makes any changes. */ void -AlterTableCreateToastTable(Oid relOid, Datum reloptions, LOCKMODE lockmode) +AlterTableCreateToastTable(Oid relOid, LOCKMODE lockmode) { - CheckAndCreateToastTable(relOid, reloptions, lockmode, true, InvalidOid); + CheckAndCreateToastTable(relOid, lockmode, true, InvalidOid); } void -NewHeapCreateToastTable(Oid relOid, Datum reloptions, LOCKMODE lockmode, - Oid OIDOldToast) +NewHeapCreateToastTable(Oid relOid, LOCKMODE lockmode, Oid OIDOldToast) { - CheckAndCreateToastTable(relOid, reloptions, lockmode, false, OIDOldToast); + CheckAndCreateToastTable(relOid, lockmode, false, OIDOldToast); } void -NewRelationCreateToastTable(Oid relOid, Datum reloptions) +NewRelationCreateToastTable(Oid relOid) { - CheckAndCreateToastTable(relOid, reloptions, AccessExclusiveLock, false, - InvalidOid); + CheckAndCreateToastTable(relOid, AccessExclusiveLock, false, InvalidOid); } static void -CheckAndCreateToastTable(Oid relOid, Datum reloptions, LOCKMODE lockmode, +CheckAndCreateToastTable(Oid relOid, LOCKMODE lockmode, bool check, Oid OIDOldToast) { Relation rel; @@ -84,7 +79,7 @@ CheckAndCreateToastTable(Oid relOid, Datum reloptions, LOCKMODE lockmode, rel = table_open(relOid, lockmode); /* create_toast_table does all the work */ - (void) create_toast_table(rel, InvalidOid, InvalidOid, reloptions, lockmode, + (void) create_toast_table(rel, InvalidOid, InvalidOid, lockmode, check, OIDOldToast); table_close(rel, NoLock); @@ -108,7 +103,7 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid) relName); /* create_toast_table does all the work */ - if (!create_toast_table(rel, toastOid, toastIndexOid, (Datum) 0, + if (!create_toast_table(rel, toastOid, toastIndexOid, AccessExclusiveLock, false, InvalidOid)) elog(ERROR, "\"%s\" does not require a toast table", relName); @@ -126,7 +121,7 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid) */ static bool create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, - Datum reloptions, LOCKMODE lockmode, bool check, + LOCKMODE lockmode, bool check, Oid OIDOldToast) { Oid relOid = RelationGetRelid(rel); @@ -266,7 +261,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, shared_relation, mapped_relation, ONCOMMIT_NOOP, - reloptions, + (Datum) 0, false, true, true, diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 6dbb831ca89..188e167999b 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -84,8 +84,6 @@ create_ctas_internal(List *attrList, IntoClause *into) CreateStmt *create = makeNode(CreateStmt); bool is_matview; char relkind; - Datum toast_options; - const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; ObjectAddress intoRelationAddr; /* This code supports both CREATE TABLE AS and CREATE MATERIALIZED VIEW */ @@ -120,16 +118,7 @@ create_ctas_internal(List *attrList, IntoClause *into) */ CommandCounterIncrement(); - /* parse and validate reloptions for the toast table */ - toast_options = transformRelOptions((Datum) 0, - create->options, - "toast", - validnsps, - true, false); - - (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); - - NewRelationCreateToastTable(intoRelationAddr.objectId, toast_options); + NewRelationCreateToastTable(intoRelationAddr.objectId); /* Create the "view" part of a materialized view. */ if (is_matview) diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 9d162957bc3..5064d8e2349 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -1201,20 +1201,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, Oid NewAccessMethod, */ toastid = OldHeap->rd_rel->reltoastrelid; if (OidIsValid(toastid)) - { - /* keep the existing toast table's reloptions, if any */ - tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(toastid)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "cache lookup failed for relation %u", toastid); - reloptions = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, - &isNull); - if (isNull) - reloptions = (Datum) 0; - - NewHeapCreateToastTable(OIDNewHeap, reloptions, lockmode, toastid); - - ReleaseSysCache(tuple); - } + NewHeapCreateToastTable(OIDNewHeap, lockmode, toastid); table_close(OldHeap, NoLock); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index d8d7969bf30..a254509cbb5 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -827,7 +827,6 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, ListCell *listptr; AttrNumber attnum; bool partitioned; - const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; Oid ofTypeId; ObjectAddress address; LOCKMODE parentLockmode; @@ -976,7 +975,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, /* * Parse and validate reloptions, if any. */ - reloptions = transformRelOptions((Datum) 0, stmt->options, NULL, validnsps, + reloptions = transformRelOptions((Datum) 0, stmt->options, NULL, NULL, true, false); switch (relkind) @@ -5433,7 +5432,7 @@ ATRewriteCatalogs(List **wqueue, LOCKMODE lockmode, tab->relkind == RELKIND_PARTITIONED_TABLE) && tab->partition_constraint == NULL) || tab->relkind == RELKIND_MATVIEW) - AlterTableCreateToastTable(tab->relid, (Datum) 0, lockmode); + AlterTableCreateToastTable(tab->relid, lockmode); } } @@ -16927,7 +16926,6 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, Datum repl_val[Natts_pg_class]; bool repl_null[Natts_pg_class]; bool repl_repl[Natts_pg_class]; - const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; if (defList == NIL && operation != AT_ReplaceRelOptions) return; /* nothing to do */ @@ -16960,7 +16958,7 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, } /* Generate new proposed reloptions (text array) */ - newOptions = transformRelOptions(datum, defList, NULL, validnsps, false, + newOptions = transformRelOptions(datum, defList, NULL, NULL, false, operation == AT_ResetRelOptions); /* Validate */ @@ -17083,20 +17081,11 @@ ATExecSetRelOptions(Relation rel, List *defList, AlterTableType operation, datum = (Datum) 0; } - newOptions = transformRelOptions(datum, defList, "toast", validnsps, - false, operation == AT_ResetRelOptions); - - (void) heap_reloptions(RELKIND_TOASTVALUE, newOptions, true); - memset(repl_val, 0, sizeof(repl_val)); memset(repl_null, false, sizeof(repl_null)); memset(repl_repl, false, sizeof(repl_repl)); - if (newOptions != (Datum) 0) - repl_val[Anum_pg_class_reloptions - 1] = newOptions; - else - repl_null[Anum_pg_class_reloptions - 1] = true; - + repl_null[Anum_pg_class_reloptions - 1] = true; repl_repl[Anum_pg_class_reloptions - 1] = true; newtuple = heap_modify_tuple(tuple, RelationGetDescr(pgclass), diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 99d0db82ed7..ed670b3e3f1 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -2015,13 +2015,6 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, Oid save_userid; int save_sec_context; int save_nestlevel; - VacuumParams toast_vacuum_params; - - /* - * This function scribbles on the parameters, so make a copy early to - * avoid affecting the TOAST table (if we do end up recursing to it). - */ - memcpy(&toast_vacuum_params, ¶ms, sizeof(VacuumParams)); /* Begin a transaction for vacuuming this relation */ StartTransactionCommand(); @@ -2336,11 +2329,10 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, * relation. NB: This is only safe to do because we hold a session * lock on the main relation that prevents concurrent deletion. */ - toast_vacuum_params.options |= VACOPT_PROCESS_MAIN; - toast_vacuum_params.toast_parent = relid; + params.options |= VACOPT_PROCESS_MAIN; + params.toast_parent = relid; - vacuum_rel(toast_relid, NULL, toast_vacuum_params, bstrategy, - isTopLevel); + vacuum_rel(toast_relid, NULL, params, bstrategy, isTopLevel); } /* diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index a5a8db2ff88..c0e7cbf2cb8 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -2139,8 +2139,9 @@ do_autovacuum(void) { Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); Oid relid; - AutoVacOpts *relopts; - bool free_relopts = false; + av_relation *hentry; + bool found; + AutoVacOpts *relopts = NULL; bool dovacuum; bool doanalyze; bool wraparound; @@ -2154,22 +2155,10 @@ do_autovacuum(void) relid = classForm->oid; - /* - * fetch reloptions -- if this toast table does not have them, try the - * main rel - */ - relopts = extract_autovac_opts(tuple, pg_class_desc); - if (relopts) - free_relopts = true; - else - { - av_relation *hentry; - bool found; - - hentry = hash_search(table_toast_map, &relid, HASH_FIND, &found); - if (found && hentry->ar_hasrelopts) - relopts = &hentry->ar_reloptions; - } + /* Use reloptions from main rel. */ + hentry = hash_search(table_toast_map, &relid, HASH_FIND, &found); + if (found && hentry->ar_hasrelopts) + relopts = &hentry->ar_reloptions; relation_needs_vacanalyze(relid, relopts, classForm, effective_multixact_freeze_max_age, @@ -2186,10 +2175,6 @@ do_autovacuum(void) table->score = scores.max; tables_to_process = lappend(tables_to_process, table); } - - /* Release stuff to avoid leakage */ - if (free_relopts) - pfree(relopts); } table_endscan(relScan); @@ -2826,7 +2811,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, bool doanalyze; autovac_table *tab = NULL; bool wraparound; - AutoVacOpts *avopts; + AutoVacOpts *avopts = NULL; bool free_avopts = false; AutoVacuumScores scores; @@ -2838,13 +2823,15 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, /* * Get the applicable reloptions. If it is a TOAST table, try to get the - * main table reloptions if the toast table itself doesn't have. + * main table reloptions. */ - avopts = extract_autovac_opts(classTup, pg_class_desc); - if (avopts) - free_avopts = true; - else if (classForm->relkind == RELKIND_TOASTVALUE && - table_toast_map != NULL) + if (classForm->relkind != RELKIND_TOASTVALUE) + { + avopts = extract_autovac_opts(classTup, pg_class_desc); + if (avopts) + free_avopts = true; + } + else if (table_toast_map) { av_relation *hentry; bool found; @@ -3127,8 +3114,7 @@ relation_needs_vacanalyze(Oid relid, /* * Determine vacuum/analyze equation parameters. We have two possible - * sources: the passed reloptions (which could be a main table or a toast - * table), or the autovacuum GUC variables. + * sources: the passed reloptions or the autovacuum GUC variables. */ /* -1 in autovac setting means use plain vacuum_scale_factor */ diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 73a56f1df1d..d089e02c216 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1159,8 +1159,6 @@ ProcessUtilitySlow(ParseState *pstate, if (IsA(stmt, CreateStmt)) { CreateStmt *cstmt = (CreateStmt *) stmt; - Datum toast_options; - const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; /* Remember transformed RangeVar for LIKE */ table_rv = cstmt->relation; @@ -1180,22 +1178,7 @@ ProcessUtilitySlow(ParseState *pstate, */ CommandCounterIncrement(); - /* - * parse and validate reloptions for the toast - * table - */ - toast_options = transformRelOptions((Datum) 0, - cstmt->options, - "toast", - validnsps, - true, - false); - (void) heap_reloptions(RELKIND_TOASTVALUE, - toast_options, - true); - - NewRelationCreateToastTable(address.objectId, - toast_options); + NewRelationCreateToastTable(address.objectId); } else if (IsA(stmt, CreateForeignTableStmt)) { diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 5a7afe62eab..f0d15f56525 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -37,6 +37,7 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(void); static void check_old_cluster_subscription_state(void); static void check_old_cluster_global_names(ClusterInfo *cluster); +static void check_for_toast_reloptions(ClusterInfo *cluster); /* * DataTypesUsageChecks - definitions of data type checks for the old cluster @@ -634,6 +635,9 @@ check_and_dump_old_cluster(void) check_for_prepared_transactions(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); + if (GET_MAJOR_VERSION(old_cluster.major_version) < 2000) + check_for_toast_reloptions(&old_cluster); + if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) { /* @@ -2644,3 +2648,74 @@ check_old_cluster_global_names(ClusterInfo *cluster) else check_ok(); } + +/* + * Callback function for processing results of query for + * check_for_toast_reloptions()'s UpgradeTask. If the query returned any rows + * (i.e., the check failed), write the details to the report file. + */ +static void +process_toast_relopts_check(DbInfo *dbinfo, PGresult *res, void *arg) +{ + UpgradeTaskReport *report = (UpgradeTaskReport *) arg; + int ntups = PQntuples(res); + int i_nspname = PQfnumber(res, "nspname"); + int i_relname = PQfnumber(res, "relname"); + + if (ntups == 0) + return; + + if (report->file == NULL && + (report->file = fopen_priv(report->path, "w")) == NULL) + pg_fatal("could not open file \"%s\": %m", report->path); + + fprintf(report->file, "In database: %s\n", dbinfo->db_name); + + for (int rowno = 0; rowno < ntups; rowno++) + fprintf(report->file, " %s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname)); +} + +/* + * Verify that no storage parameters (a.k.a. reloptions) are defined for TOAST + * tables. + */ +static void +check_for_toast_reloptions(ClusterInfo *cluster) +{ + UpgradeTaskReport report; + UpgradeTask *task = upgrade_task_create(); + const char *query = "SELECT n.nspname, c.relname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_class tc, " + " pg_catalog.pg_namespace n " + "WHERE c.reltoastrelid = tc.oid AND " + " c.relnamespace = c.oid AND " + " tc.reloptions IS NOT NULL"; + + prep_status("Check for tables with TOAST storage parameters"); + + report.file = NULL; + snprintf(report.path, sizeof(report.path), "%s/%s", + log_opts.basedir, + "tables_with_toast_storage_parameters.txt"); + + upgrade_task_add_step(task, query, process_toast_relopts_check, + true, &report); + upgrade_task_run(task, cluster); + upgrade_task_free(task); + + if (report.file) + { + fclose(report.file); + pg_log(PG_REPORT, "fatal"); + pg_fatal("Your installation contains tables with TOAST storage parameters set, which is\n" + "not supported anymore. Consider remove the TOAST storage parameters using\n" + " ALTER TABLE ... RESET ( ... );\n" + "A list of tables with the problem is in the file:\n" + " %s", report.path); + } + else + check_ok(); +} diff --git a/src/include/access/reloptions.h b/src/include/access/reloptions.h index e8cb7f7a627..4415c1ca9b5 100644 --- a/src/include/access/reloptions.h +++ b/src/include/access/reloptions.h @@ -40,26 +40,22 @@ typedef enum relopt_kind { RELOPT_KIND_LOCAL = 0, RELOPT_KIND_HEAP = (1 << 0), - RELOPT_KIND_TOAST = (1 << 1), - RELOPT_KIND_BTREE = (1 << 2), - RELOPT_KIND_HASH = (1 << 3), - RELOPT_KIND_GIN = (1 << 4), - RELOPT_KIND_GIST = (1 << 5), - RELOPT_KIND_ATTRIBUTE = (1 << 6), - RELOPT_KIND_TABLESPACE = (1 << 7), - RELOPT_KIND_SPGIST = (1 << 8), - RELOPT_KIND_VIEW = (1 << 9), - RELOPT_KIND_BRIN = (1 << 10), - RELOPT_KIND_PARTITIONED = (1 << 11), + RELOPT_KIND_BTREE = (1 << 1), + RELOPT_KIND_HASH = (1 << 2), + RELOPT_KIND_GIN = (1 << 3), + RELOPT_KIND_GIST = (1 << 4), + RELOPT_KIND_ATTRIBUTE = (1 << 5), + RELOPT_KIND_TABLESPACE = (1 << 6), + RELOPT_KIND_SPGIST = (1 << 7), + RELOPT_KIND_VIEW = (1 << 8), + RELOPT_KIND_BRIN = (1 << 9), + RELOPT_KIND_PARTITIONED = (1 << 10), /* if you add a new kind, make sure you update "last_default" too */ RELOPT_KIND_LAST_DEFAULT = RELOPT_KIND_PARTITIONED, /* some compilers treat enums as signed ints, so we can't use 1 << 31 */ RELOPT_KIND_MAX = (1 << 30) } relopt_kind; -/* reloption namespaces allowed for heaps -- currently only TOAST */ -#define HEAP_RELOPT_NAMESPACES { "toast", NULL } - /* generic struct to hold shared data */ typedef struct relopt_gen { diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index 0bc61a8fee9..13e5a8affcb 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -19,10 +19,10 @@ /* * toasting.c prototypes */ -extern void NewRelationCreateToastTable(Oid relOid, Datum reloptions); -extern void NewHeapCreateToastTable(Oid relOid, Datum reloptions, +extern void NewRelationCreateToastTable(Oid relOid); +extern void NewHeapCreateToastTable(Oid relOid, LOCKMODE lockmode, Oid OIDOldToast); -extern void AlterTableCreateToastTable(Oid relOid, Datum reloptions, +extern void AlterTableCreateToastTable(Oid relOid, LOCKMODE lockmode); extern void BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid); diff --git a/src/test/modules/injection_points/expected/vacuum.out b/src/test/modules/injection_points/expected/vacuum.out index 58df59fa927..dcfa729d05e 100644 --- a/src/test/modules/injection_points/expected/vacuum.out +++ b/src/test/modules/injection_points/expected/vacuum.out @@ -43,29 +43,29 @@ SELECT injection_points_attach('vacuum-truncate-enabled', 'notice'); (1 row) -- Check state of index_cleanup and truncate in VACUUM. -CREATE TABLE vac_tab_on_toast_off(i int, j text) WITH +CREATE TABLE vac_tab_on_toast(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=true, toast.vacuum_index_cleanup=false, - vacuum_truncate=true, toast.vacuum_truncate=false); -CREATE TABLE vac_tab_off_toast_on(i int, j text) WITH + vacuum_index_cleanup=true, + vacuum_truncate=true); +CREATE TABLE vac_tab_off_toast(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=false, toast.vacuum_index_cleanup=true, - vacuum_truncate=false, toast.vacuum_truncate=true); + vacuum_index_cleanup=false, + vacuum_truncate=false); -- Multiple relations should use their options in isolation. -VACUUM vac_tab_on_toast_off, vac_tab_off_toast_on; +VACUUM vac_tab_on_toast, vac_tab_off_toast; +NOTICE: notice triggered for injection point vacuum-index-cleanup-enabled +NOTICE: notice triggered for injection point vacuum-truncate-enabled NOTICE: notice triggered for injection point vacuum-index-cleanup-enabled NOTICE: notice triggered for injection point vacuum-truncate-enabled NOTICE: notice triggered for injection point vacuum-index-cleanup-disabled NOTICE: notice triggered for injection point vacuum-truncate-disabled NOTICE: notice triggered for injection point vacuum-index-cleanup-disabled NOTICE: notice triggered for injection point vacuum-truncate-disabled -NOTICE: notice triggered for injection point vacuum-index-cleanup-enabled -NOTICE: notice triggered for injection point vacuum-truncate-enabled -- Check "auto" case of index_cleanup and "truncate" controlled by -- its GUC. CREATE TABLE vac_tab_auto(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=auto, toast.vacuum_index_cleanup=auto); + vacuum_index_cleanup=auto); SET vacuum_truncate = false; VACUUM vac_tab_auto; NOTICE: notice triggered for injection point vacuum-index-cleanup-auto @@ -80,8 +80,8 @@ NOTICE: notice triggered for injection point vacuum-index-cleanup-auto NOTICE: notice triggered for injection point vacuum-truncate-enabled RESET vacuum_truncate; DROP TABLE vac_tab_auto; -DROP TABLE vac_tab_on_toast_off; -DROP TABLE vac_tab_off_toast_on; +DROP TABLE vac_tab_on_toast; +DROP TABLE vac_tab_off_toast; -- Cleanup SELECT injection_points_detach('vacuum-index-cleanup-auto'); injection_points_detach diff --git a/src/test/modules/injection_points/sql/vacuum.sql b/src/test/modules/injection_points/sql/vacuum.sql index 23760dd0f38..ee7a2e0566e 100644 --- a/src/test/modules/injection_points/sql/vacuum.sql +++ b/src/test/modules/injection_points/sql/vacuum.sql @@ -11,22 +11,22 @@ SELECT injection_points_attach('vacuum-truncate-disabled', 'notice'); SELECT injection_points_attach('vacuum-truncate-enabled', 'notice'); -- Check state of index_cleanup and truncate in VACUUM. -CREATE TABLE vac_tab_on_toast_off(i int, j text) WITH +CREATE TABLE vac_tab_on_toast(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=true, toast.vacuum_index_cleanup=false, - vacuum_truncate=true, toast.vacuum_truncate=false); -CREATE TABLE vac_tab_off_toast_on(i int, j text) WITH + vacuum_index_cleanup=true, + vacuum_truncate=true); +CREATE TABLE vac_tab_off_toast(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=false, toast.vacuum_index_cleanup=true, - vacuum_truncate=false, toast.vacuum_truncate=true); + vacuum_index_cleanup=false, + vacuum_truncate=false); -- Multiple relations should use their options in isolation. -VACUUM vac_tab_on_toast_off, vac_tab_off_toast_on; +VACUUM vac_tab_on_toast, vac_tab_off_toast; -- Check "auto" case of index_cleanup and "truncate" controlled by -- its GUC. CREATE TABLE vac_tab_auto(i int, j text) WITH (autovacuum_enabled=false, - vacuum_index_cleanup=auto, toast.vacuum_index_cleanup=auto); + vacuum_index_cleanup=auto); SET vacuum_truncate = false; VACUUM vac_tab_auto; SET vacuum_truncate = true; @@ -34,8 +34,8 @@ VACUUM vac_tab_auto; RESET vacuum_truncate; DROP TABLE vac_tab_auto; -DROP TABLE vac_tab_on_toast_off; -DROP TABLE vac_tab_off_toast_on; +DROP TABLE vac_tab_on_toast; +DROP TABLE vac_tab_off_toast; -- Cleanup SELECT injection_points_detach('vacuum-index-cleanup-auto'); diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out index 6dd22be0e8d..435e6beea0f 100644 --- a/src/test/regress/expected/alter_table.out +++ b/src/test/regress/expected/alter_table.out @@ -2860,15 +2860,6 @@ select * from my_locks order by 1; pg_toast | ShareUpdateExclusiveLock (2 rows) -commit; -begin; alter table alterlock set (toast.autovacuum_enabled = off); -select * from my_locks order by 1; - relname | max_lockmode ------------+-------------------------- - alterlock | ShareUpdateExclusiveLock - pg_toast | ShareUpdateExclusiveLock -(2 rows) - commit; begin; alter table alterlock set (autovacuum_enabled = off); select * from my_locks order by 1; diff --git a/src/test/regress/expected/reloptions.out b/src/test/regress/expected/reloptions.out index e3a974f2611..eff5fbece60 100644 --- a/src/test/regress/expected/reloptions.out +++ b/src/test/regress/expected/reloptions.out @@ -120,7 +120,6 @@ SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test'::regclass; DROP TABLE reloptions_test; CREATE TEMP TABLE reloptions_test(i INT NOT NULL, j text) WITH (vacuum_truncate=false, - toast.vacuum_truncate=false, autovacuum_enabled=false); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test'::regclass; reloptions @@ -142,9 +141,9 @@ SELECT pg_relation_size('reloptions_test') > 0; SELECT reloptions FROM pg_class WHERE oid = (SELECT reltoastrelid FROM pg_class WHERE oid = 'reloptions_test'::regclass); - reloptions -------------------------- - {vacuum_truncate=false} + reloptions +------------ + (1 row) ALTER TABLE reloptions_test RESET (vacuum_truncate); @@ -165,56 +164,11 @@ SELECT pg_relation_size('reloptions_test') = 0; t (1 row) --- Test toast.* options -DROP TABLE reloptions_test; -CREATE TABLE reloptions_test (s VARCHAR) - WITH (toast.autovacuum_vacuum_cost_delay = 23); -SELECT reltoastrelid as toast_oid - FROM pg_class WHERE oid = 'reloptions_test'::regclass \gset -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - reloptions ------------------------------------ - {autovacuum_vacuum_cost_delay=23} -(1 row) - -ALTER TABLE reloptions_test SET (toast.autovacuum_vacuum_cost_delay = 24); -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - reloptions ------------------------------------ - {autovacuum_vacuum_cost_delay=24} -(1 row) - -ALTER TABLE reloptions_test RESET (toast.autovacuum_vacuum_cost_delay); -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - reloptions ------------- - -(1 row) - --- Fail on non-existent options in toast namespace -CREATE TABLE reloptions_test2 (i int) WITH (toast.not_existing_option = 42); -ERROR: unrecognized parameter "not_existing_option" --- Mix TOAST & heap -DROP TABLE reloptions_test; -CREATE TABLE reloptions_test (s VARCHAR) WITH - (toast.autovacuum_vacuum_cost_delay = 23, - autovacuum_vacuum_cost_delay = 24, fillfactor = 40); -SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test'::regclass; - reloptions -------------------------------------------------- - {autovacuum_vacuum_cost_delay=24,fillfactor=40} -(1 row) - -SELECT reloptions FROM pg_class WHERE oid = ( - SELECT reltoastrelid FROM pg_class WHERE oid = 'reloptions_test'::regclass); - reloptions ------------------------------------ - {autovacuum_vacuum_cost_delay=23} -(1 row) - -- -- CREATE INDEX, ALTER INDEX for btrees -- +DROP TABLE reloptions_test; +CREATE TABLE reloptions_test (s VARCHAR); CREATE INDEX reloptions_test_idx ON reloptions_test (s) WITH (fillfactor=30); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test_idx'::regclass; reloptions diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index d4696bc3325..d551ef57d50 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -209,14 +209,6 @@ VACUUM no_index_cleanup; INSERT INTO no_index_cleanup(i, t) VALUES (generate_series(31,60), repeat('1234567890',269)); DELETE FROM no_index_cleanup WHERE i < 45; --- Only toast index is cleaned up. -ALTER TABLE no_index_cleanup SET (vacuum_index_cleanup = off, - toast.vacuum_index_cleanup = yes); -VACUUM no_index_cleanup; --- Only parent is cleaned up. -ALTER TABLE no_index_cleanup SET (vacuum_index_cleanup = true, - toast.vacuum_index_cleanup = false); -VACUUM no_index_cleanup; -- Test some extra relations. VACUUM (INDEX_CLEANUP FALSE) vaccluster; VACUUM (INDEX_CLEANUP AUTO) vactst; -- index cleanup option is ignored if no indexes diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql index f5f13bbd3e7..410465dc022 100644 --- a/src/test/regress/sql/alter_table.sql +++ b/src/test/regress/sql/alter_table.sql @@ -1784,10 +1784,6 @@ begin; alter table alterlock reset (fillfactor); select * from my_locks order by 1; commit; -begin; alter table alterlock set (toast.autovacuum_enabled = off); -select * from my_locks order by 1; -commit; - begin; alter table alterlock set (autovacuum_enabled = off); select * from my_locks order by 1; commit; diff --git a/src/test/regress/sql/reloptions.sql b/src/test/regress/sql/reloptions.sql index 680c8bf8614..0809a2ae0e4 100644 --- a/src/test/regress/sql/reloptions.sql +++ b/src/test/regress/sql/reloptions.sql @@ -75,7 +75,6 @@ DROP TABLE reloptions_test; CREATE TEMP TABLE reloptions_test(i INT NOT NULL, j text) WITH (vacuum_truncate=false, - toast.vacuum_truncate=false, autovacuum_enabled=false); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test'::regclass; INSERT INTO reloptions_test VALUES (1, NULL), (NULL, NULL); @@ -94,39 +93,12 @@ INSERT INTO reloptions_test VALUES (1, NULL), (NULL, NULL); VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) reloptions_test; SELECT pg_relation_size('reloptions_test') = 0; --- Test toast.* options -DROP TABLE reloptions_test; - -CREATE TABLE reloptions_test (s VARCHAR) - WITH (toast.autovacuum_vacuum_cost_delay = 23); -SELECT reltoastrelid as toast_oid - FROM pg_class WHERE oid = 'reloptions_test'::regclass \gset -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - -ALTER TABLE reloptions_test SET (toast.autovacuum_vacuum_cost_delay = 24); -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - -ALTER TABLE reloptions_test RESET (toast.autovacuum_vacuum_cost_delay); -SELECT reloptions FROM pg_class WHERE oid = :toast_oid; - --- Fail on non-existent options in toast namespace -CREATE TABLE reloptions_test2 (i int) WITH (toast.not_existing_option = 42); - --- Mix TOAST & heap -DROP TABLE reloptions_test; - -CREATE TABLE reloptions_test (s VARCHAR) WITH - (toast.autovacuum_vacuum_cost_delay = 23, - autovacuum_vacuum_cost_delay = 24, fillfactor = 40); - -SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test'::regclass; -SELECT reloptions FROM pg_class WHERE oid = ( - SELECT reltoastrelid FROM pg_class WHERE oid = 'reloptions_test'::regclass); - -- -- CREATE INDEX, ALTER INDEX for btrees -- +DROP TABLE reloptions_test; +CREATE TABLE reloptions_test (s VARCHAR); CREATE INDEX reloptions_test_idx ON reloptions_test (s) WITH (fillfactor=30); SELECT reloptions FROM pg_class WHERE oid = 'reloptions_test_idx'::regclass; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 247b8e23b23..37a84fd9145 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -175,14 +175,6 @@ VACUUM no_index_cleanup; INSERT INTO no_index_cleanup(i, t) VALUES (generate_series(31,60), repeat('1234567890',269)); DELETE FROM no_index_cleanup WHERE i < 45; --- Only toast index is cleaned up. -ALTER TABLE no_index_cleanup SET (vacuum_index_cleanup = off, - toast.vacuum_index_cleanup = yes); -VACUUM no_index_cleanup; --- Only parent is cleaned up. -ALTER TABLE no_index_cleanup SET (vacuum_index_cleanup = true, - toast.vacuum_index_cleanup = false); -VACUUM no_index_cleanup; -- Test some extra relations. VACUUM (INDEX_CLEANUP FALSE) vaccluster; VACUUM (INDEX_CLEANUP AUTO) vactst; -- index cleanup option is ignored if no indexes -- 2.50.1 (Apple Git-155) --UOGHeidpVrTAfMhN-- ^ permalink raw reply [nested|flat] 23+ messages in thread
end of thread, other threads:[~2026-04-07 20:30 UTC | newest] Thread overview: 23+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2024-11-25 21:35 [PATCH v2 14/20] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2024-11-25 21:35 [PATCH v2 14/20] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2024-11-25 21:35 [PATCH v2 14/20] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-01-22 22:13 [PATCH v2.3 23/30] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-01-22 22:13 [PATCH v2.3 23/30] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-02-18 01:03 [PATCH v2.5 23/30] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-02-18 01:03 [PATCH v2.4 23/29] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-02-18 01:03 [PATCH v2.4 23/29] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-07 20:47 [PATCH v2.7 28/35] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-07 20:47 [PATCH v2.6 27/34] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-07 20:47 [PATCH v2.8 30/38] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-07 20:47 [PATCH v2.8 30/38] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-15 16:29 [PATCH v2.9 22/30] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.11 20/27] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.15 10/18] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.13 20/28] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.10 19/28] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.14 21/29] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.13 20/28] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.10 19/28] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.11 20/27] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2025-03-18 18:40 [PATCH v2.12 20/28] aio: Add bounce buffers Andres Freund <andres@anarazel.de> 2026-04-07 20:30 [PATCH v3 1/1] remove toast reloptions Nathan Bossart <nathan@postgresql.org>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox