From: Andres Freund Date: Mon, 25 Nov 2024 16:35:15 -0500 Subject: [PATCH v2 14/20] aio: Add bounce buffers --- src/include/storage/aio.h | 18 ++ src/include/storage/aio_internal.h | 33 ++++ src/include/utils/resowner.h | 2 + src/backend/storage/aio/README.md | 27 +++ src/backend/storage/aio/aio.c | 182 ++++++++++++++++++ src/backend/storage/aio/aio_init.c | 118 ++++++++++++ src/backend/utils/misc/guc_tables.c | 13 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/backend/utils/resowner/resowner.c | 25 ++- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 419 insertions(+), 2 deletions(-) diff --git a/src/include/storage/aio.h b/src/include/storage/aio.h index ff44dac5bb2..1bef475b0a9 100644 --- a/src/include/storage/aio.h +++ b/src/include/storage/aio.h @@ -222,6 +222,9 @@ typedef struct PgAioHandleSharedCallbacks +typedef struct PgAioBounceBuffer PgAioBounceBuffer; + + /* * How many callbacks can be registered for one IO handle. Currently we only * need two, but it's not hard to imagine needing a few more. @@ -294,6 +297,20 @@ extern void pgaio_result_log(PgAioResult result, const PgAioSubjectData *subject +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +extern PgAioBounceBuffer *pgaio_bounce_buffer_get(void); +extern void pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb); +extern uint32 pgaio_bounce_buffer_id(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release(PgAioBounceBuffer *bb); +extern char *pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb); +extern void pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error); + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -354,6 +371,7 @@ typedef enum IoMethod extern const struct config_enum_entry io_method_options[]; extern int io_method; extern int io_max_concurrency; +extern int io_bounce_buffers; #endif /* AIO_H */ diff --git a/src/include/storage/aio_internal.h b/src/include/storage/aio_internal.h index d2dc1516bdf..2065bde79c3 100644 --- a/src/include/storage/aio_internal.h +++ b/src/include/storage/aio_internal.h @@ -91,6 +91,12 @@ struct PgAioHandle /* index into PgAioCtl->iovecs */ uint32 iovec_off; + /* + * List of bounce_buffers owned by IO. It would suffice to use an index + * based linked list here. + */ + slist_head bounce_buffers; + /** * In which list the handle is registered, depends on the state: * - IDLE, in per-backend list @@ -130,11 +136,23 @@ struct PgAioHandle }; +struct PgAioBounceBuffer +{ + slist_node node; + struct ResourceOwnerData *resowner; + dlist_node resowner_node; + char *buffer; +}; + + typedef struct PgAioPerBackend { /* index into PgAioCtl->io_handles */ uint32 io_handle_off; + /* index into PgAioCtl->bounce_buffers */ + uint32 bounce_buffers_off; + /* IO Handles that currently are not used */ dclist_head idle_ios; @@ -162,6 +180,12 @@ typedef struct PgAioPerBackend * IOs being appended at the end. */ dclist_head in_flight_ios; + + /* Bounce Buffers that currently are not used */ + slist_head idle_bbs; + + /* see handed_out_io */ + PgAioBounceBuffer *handed_out_bb; } PgAioPerBackend; @@ -187,6 +211,15 @@ typedef struct PgAioCtl */ uint64 *iovecs_data; + /* + * To perform AIO on buffers that are not located in shared memory (either + * because they are not in shared memory or because we need to operate on + * a copy, as e.g. the case for writes when checksums are in use) + */ + uint64 bounce_buffers_count; + PgAioBounceBuffer *bounce_buffers; + char *bounce_buffers_data; + uint64 io_handle_count; PgAioHandle *io_handles; } PgAioCtl; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 2d55720a54c..0cdd0c13ffb 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -168,5 +168,7 @@ extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *local struct dlist_node; extern void ResourceOwnerRememberAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); extern void ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node); +extern void ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); +extern void ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *bb_node); #endif /* RESOWNER_H */ diff --git a/src/backend/storage/aio/README.md b/src/backend/storage/aio/README.md index 893f4ffe428..0076ea4aa10 100644 --- a/src/backend/storage/aio/README.md +++ b/src/backend/storage/aio/README.md @@ -395,6 +395,33 @@ shared memory no less!), completion callbacks instead have to encode errors in a more compact format that can be converted into an error message. +### AIO Bounce Buffers + +For some uses of AIO there is no convenient memory location as the source / +destination of an AIO. E.g. when data checksums are enabled, writes from +shared buffers currently cannot be done directly from shared buffers, as a +shared buffer lock still allows some modification, e.g., for hint bits(see +`FlushBuffer()`). If the write were done in-place, such modifications can +cause the checksum to fail. + +For synchronous IO this is solved by copying the buffer to separate memory +before computing the checksum and using that copy as the source buffer for the +AIO. + +However, for AIO that is not a workable solution: +- Instead of a single buffer many buffers are required, as many IOs might be + in flight +- When using the [worker method](#worker), the source/target of IO needs to be + in shared memory, otherwise the workers won't be able to access the memory. + +The AIO subsystem addresses this by providing a limited number of bounce +buffers that can be used as the source / target for IO. A bounce buffer be +acquired with `pgaio_bounce_buffer_get()` and multiple bounce buffers can be +associated with an AIO Handle with `pgaio_io_assoc_bounce_buffer()`. + +Bounce buffers are automatically released when the IO completes. + + ## Helpers Using the low-level AIO API introduces too much complexity to do so all over diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 2439ce3740d..e829e1752ca 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -54,6 +54,8 @@ static void pgaio_io_resowner_register(PgAioHandle *ioh); static void pgaio_io_wait_for_free(void); static PgAioHandle *pgaio_io_from_ref(PgAioHandleRef *ior, uint64 *ref_generation); +static void pgaio_bounce_buffer_wait_for_free(void); + /* Options for io_method. */ @@ -68,6 +70,7 @@ const struct config_enum_entry io_method_options[] = { int io_method = DEFAULT_IO_METHOD; int io_max_concurrency = -1; +int io_bounce_buffers = -1; /* global control for AIO */ @@ -732,6 +735,21 @@ pgaio_io_reclaim(PgAioHandle *ioh) } } + /* reclaim all associated bounce buffers */ + if (!slist_is_empty(&ioh->bounce_buffers)) + { + slist_mutable_iter it; + + slist_foreach_modify(it, &ioh->bounce_buffers) + { + PgAioBounceBuffer *bb = slist_container(PgAioBounceBuffer, node, it.cur); + + slist_delete_current(&it); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + } + } + if (ioh->resowner) { ResourceOwnerForgetAioHandle(ioh->resowner, &ioh->resowner_node); @@ -855,6 +873,168 @@ pgaio_io_wait_for_free(void) +/* -------------------------------------------------------------------------------- + * Bounce Buffers + * -------------------------------------------------------------------------------- + */ + +PgAioBounceBuffer * +pgaio_bounce_buffer_get(void) +{ + PgAioBounceBuffer *bb = NULL; + slist_node *node; + + if (my_aio->handed_out_bb != NULL) + elog(ERROR, "can only hand out one BB"); + + /* + * FIXME It probably is not correct to have bounce buffers be per backend, + * they use too much memory. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + { + pgaio_bounce_buffer_wait_for_free(); + } + + node = slist_pop_head_node(&my_aio->idle_bbs); + bb = slist_container(PgAioBounceBuffer, node, node); + + my_aio->handed_out_bb = bb; + + bb->resowner = CurrentResourceOwner; + ResourceOwnerRememberAioBounceBuffer(bb->resowner, &bb->resowner_node); + + return bb; +} + +void +pgaio_io_assoc_bounce_buffer(PgAioHandle *ioh, PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only assign handed out BB"); + my_aio->handed_out_bb = NULL; + + /* + * There can be many bounce buffers assigned in case of vectorized IOs. + */ + slist_push_head(&ioh->bounce_buffers, &bb->node); + + /* once associated with an IO, the IO has ownership */ + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +uint32 +pgaio_bounce_buffer_id(PgAioBounceBuffer *bb) +{ + return bb - aio_ctl->bounce_buffers; +} + +void +pgaio_bounce_buffer_release(PgAioBounceBuffer *bb) +{ + if (my_aio->handed_out_bb != bb) + elog(ERROR, "can only release handed out BB"); + + slist_push_head(&my_aio->idle_bbs, &bb->node); + my_aio->handed_out_bb = NULL; + + ResourceOwnerForgetAioBounceBuffer(bb->resowner, &bb->resowner_node); + bb->resowner = NULL; +} + +void +pgaio_bounce_buffer_release_resowner(dlist_node *bb_node, bool on_error) +{ + PgAioBounceBuffer *bb = dlist_container(PgAioBounceBuffer, resowner_node, bb_node); + + Assert(bb->resowner); + + if (!on_error) + elog(WARNING, "leaked AIO bounce buffer"); + + pgaio_bounce_buffer_release(bb); +} + +char * +pgaio_bounce_buffer_buffer(PgAioBounceBuffer *bb) +{ + return bb->buffer; +} + +static void +pgaio_bounce_buffer_wait_for_free(void) +{ + static uint32 lastpos = 0; + + if (my_aio->num_staged_ios > 0) + { + elog(DEBUG2, "submitting while acquiring free bb"); + pgaio_submit_staged(); + } + + for (uint32 i = lastpos; i < lastpos + io_max_concurrency; i++) + { + uint32 thisoff = my_aio->io_handle_off + (i % io_max_concurrency); + PgAioHandle *ioh = &aio_ctl->io_handles[thisoff]; + + switch (ioh->state) + { + case AHS_IDLE: + case AHS_HANDED_OUT: + continue; + case AHS_DEFINED: /* should have been submitted above */ + case AHS_PREPARED: + elog(ERROR, "shouldn't get here with io:%d in state %d", + pgaio_io_get_id(ioh), ioh->state); + break; + case AHS_REAPED: + case AHS_IN_FLIGHT: + if (!slist_is_empty(&ioh->bounce_buffers)) + { + PgAioHandleRef ior; + + ior.aio_index = ioh - aio_ctl->io_handles; + ior.generation_upper = (uint32) (ioh->generation >> 32); + ior.generation_lower = (uint32) ioh->generation; + + pgaio_io_ref_wait(&ior); + elog(DEBUG2, "waited for io:%d to reclaim BB", + pgaio_io_get_id(ioh)); + + if (slist_is_empty(&my_aio->idle_bbs)) + elog(WARNING, "empty after wait"); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + } + break; + case AHS_COMPLETED_SHARED: + case AHS_COMPLETED_LOCAL: + /* reclaim */ + pgaio_io_reclaim(ioh); + + if (!slist_is_empty(&my_aio->idle_bbs)) + { + lastpos = i; + return; + } + break; + } + } + + /* + * The submission above could have caused the IO to complete at any time. + */ + if (slist_is_empty(&my_aio->idle_bbs)) + elog(PANIC, "no more bbs"); +} + + + /* -------------------------------------------------------------------------------- * Actions on multiple IOs. * -------------------------------------------------------------------------------- @@ -929,6 +1109,7 @@ void pgaio_at_xact_end(bool is_subxact, bool is_commit) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } /* @@ -939,6 +1120,7 @@ void pgaio_at_error(void) { Assert(!my_aio->handed_out_io); + Assert(!my_aio->handed_out_bb); } diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index 23adc5308e5..417526f3823 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -82,6 +82,32 @@ AioIOVDataShmemSize(void) io_max_concurrency)); } +static Size +AioBounceBufferDescShmemSize(void) +{ + Size sz; + + /* PgAioBounceBuffer itself */ + sz = mul_size(sizeof(PgAioBounceBuffer), + mul_size(AioProcs(), io_bounce_buffers)); + + return sz; +} + +static Size +AioBounceBufferDataShmemSize(void) +{ + Size sz; + + /* and the associated buffer */ + sz = mul_size(BLCKSZ, + mul_size(io_bounce_buffers, AioProcs())); + /* memory for alignment */ + sz += BLCKSZ; + + return sz; +} + /* * Choose a suitable value for io_max_concurrency. * @@ -107,6 +133,33 @@ AioChooseMaxConccurrency(void) return Min(max_proportional_pins, 64); } +/* + * Choose a suitable value for io_bounce_buffers. + * + * It's very unlikely to be useful to allocate more bounce buffers for each + * backend than the backend is allowed to pin. Additionally, bounce buffers + * currently are used for writes, it seems very uncommon for more than 10% of + * shared_buffers to be written out concurrently. + * + * XXX: This quickly can take up significant amounts of memory, the logic + * should probably fine tuned. + */ +static int +AioChooseBounceBuffers(void) +{ + uint32 max_backends; + int max_proportional_pins; + + /* Similar logic to LimitAdditionalPins() */ + max_backends = MaxBackends + NUM_AUXILIARY_PROCS; + max_proportional_pins = (NBuffers / 10) / max_backends; + + max_proportional_pins = Max(max_proportional_pins, 1); + + /* apply upper limit */ + return Min(max_proportional_pins, 256); +} + Size AioShmemSize(void) { @@ -130,11 +183,31 @@ AioShmemSize(void) PGC_S_OVERRIDE); } + + /* + * If io_bounce_buffers is -1, we automatically choose a suitable value. + * + * See also comment above. + */ + if (io_bounce_buffers == -1) + { + char buf[32]; + + snprintf(buf, sizeof(buf), "%d", AioChooseBounceBuffers()); + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_DYNAMIC_DEFAULT); + if (io_bounce_buffers == -1) /* failed to apply it? */ + SetConfigOption("io_bounce_buffers", buf, PGC_POSTMASTER, + PGC_S_OVERRIDE); + } + sz = add_size(sz, AioCtlShmemSize()); sz = add_size(sz, AioBackendShmemSize()); sz = add_size(sz, AioHandleShmemSize()); sz = add_size(sz, AioIOVShmemSize()); sz = add_size(sz, AioIOVDataShmemSize()); + sz = add_size(sz, AioBounceBufferDescShmemSize()); + sz = add_size(sz, AioBounceBufferDataShmemSize()); if (pgaio_impl->shmem_size) sz = add_size(sz, pgaio_impl->shmem_size()); @@ -148,7 +221,10 @@ AioShmemInit(void) bool found; uint32 io_handle_off = 0; uint32 iovec_off = 0; + uint32 bounce_buffers_off = 0; uint32 per_backend_iovecs = io_max_concurrency * io_combine_limit; + uint32 per_backend_bb = io_bounce_buffers; + char *bounce_buffers_data; aio_ctl = (PgAioCtl *) ShmemInitStruct("AioCtl", AioCtlShmemSize(), &found); @@ -160,6 +236,7 @@ AioShmemInit(void) aio_ctl->io_handle_count = AioProcs() * io_max_concurrency; aio_ctl->iovec_count = AioProcs() * per_backend_iovecs; + aio_ctl->bounce_buffers_count = AioProcs() * per_backend_bb; aio_ctl->backend_state = (PgAioPerBackend *) ShmemInitStruct("AioBackend", AioBackendShmemSize(), &found); @@ -170,6 +247,35 @@ AioShmemInit(void) aio_ctl->iovecs = ShmemInitStruct("AioIOV", AioIOVShmemSize(), &found); aio_ctl->iovecs_data = ShmemInitStruct("AioIOVData", AioIOVDataShmemSize(), &found); + aio_ctl->bounce_buffers = ShmemInitStruct("AioBounceBufferDesc", AioBounceBufferDescShmemSize(), &found); + + bounce_buffers_data = ShmemInitStruct("AioBounceBufferData", AioBounceBufferDataShmemSize(), &found); + bounce_buffers_data = (char *) TYPEALIGN(BLCKSZ, (uintptr_t) bounce_buffers_data); + aio_ctl->bounce_buffers_data = bounce_buffers_data; + + + /* Initialize IO handles. */ + for (uint64 i = 0; i < aio_ctl->io_handle_count; i++) + { + PgAioHandle *ioh = &aio_ctl->io_handles[i]; + + ioh->op = PGAIO_OP_INVALID; + ioh->subject = ASI_INVALID; + ioh->state = AHS_IDLE; + + slist_init(&ioh->bounce_buffers); + } + + /* Initialize Bounce Buffers. */ + for (uint64 i = 0; i < aio_ctl->bounce_buffers_count; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[i]; + + bb->buffer = bounce_buffers_data; + bounce_buffers_data += BLCKSZ; + } + + for (int procno = 0; procno < AioProcs(); procno++) { PgAioPerBackend *bs = &aio_ctl->backend_state[procno]; @@ -177,9 +283,13 @@ AioShmemInit(void) bs->io_handle_off = io_handle_off; io_handle_off += io_max_concurrency; + bs->bounce_buffers_off = bounce_buffers_off; + bounce_buffers_off += per_backend_bb; + dclist_init(&bs->idle_ios); memset(bs->staged_ios, 0, sizeof(PgAioHandle *) * PGAIO_SUBMIT_BATCH_SIZE); dclist_init(&bs->in_flight_ios); + slist_init(&bs->idle_bbs); /* initialize per-backend IOs */ for (int i = 0; i < io_max_concurrency; i++) @@ -201,6 +311,14 @@ AioShmemInit(void) dclist_push_tail(&bs->idle_ios, &ioh->node); iovec_off += io_combine_limit; } + + /* initialize per-backend bounce buffers */ + for (int i = 0; i < per_backend_bb; i++) + { + PgAioBounceBuffer *bb = &aio_ctl->bounce_buffers[bs->bounce_buffers_off + i]; + + slist_push_head(&bs->idle_bbs, &bb->node); + } } out: diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b2999b86c24..39e91ebd2a5 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3233,6 +3233,19 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"io_bounce_buffers", + PGC_POSTMASTER, + RESOURCES_ASYNCHRONOUS, + gettext_noop("Number of IO Bounce Buffers reserved for each backend."), + NULL, + GUC_UNIT_BLOCKS + }, + &io_bounce_buffers, + -1, -1, 4096, + NULL, NULL, NULL + }, + { {"io_workers", PGC_SIGHUP, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5893eb29228..da6e248a29e 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -848,6 +848,8 @@ #io_max_concurrency = 32 # Max number of IOs that may be in # flight at the same time in one backend # (change requires restart) +#io_bounce_buffers = -1 # -1 sets based on shared_buffers + # (change requires restart) #------------------------------------------------------------------------------ diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 5cf14472ebd..d1932b7393c 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -159,10 +159,11 @@ struct ResourceOwnerData LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ /* - * AIO handles need be registered in critical sections and therefore - * cannot use the normal ResoureElem mechanism. + * AIO handles & bounce buffers need be registered in critical sections + * and therefore cannot use the normal ResoureElem mechanism. */ dlist_head aio_handles; + dlist_head aio_bounce_buffers; }; @@ -434,6 +435,7 @@ ResourceOwnerCreate(ResourceOwner parent, const char *name) } dlist_init(&owner->aio_handles); + dlist_init(&owner->aio_bounce_buffers); return owner; } @@ -743,6 +745,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, pgaio_io_release_resowner(node, !isCommit); } + + while (!dlist_is_empty(&owner->aio_bounce_buffers)) + { + dlist_node *node = dlist_head_node(&owner->aio_bounce_buffers); + + pgaio_bounce_buffer_release_resowner(node, !isCommit); + } } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -1112,3 +1121,15 @@ ResourceOwnerForgetAioHandle(ResourceOwner owner, struct dlist_node *ioh_node) { dlist_delete_from(&owner->aio_handles, ioh_node); } + +void +ResourceOwnerRememberAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_push_tail(&owner->aio_bounce_buffers, ioh_node); +} + +void +ResourceOwnerForgetAioBounceBuffer(ResourceOwner owner, struct dlist_node *ioh_node) +{ + dlist_delete_from(&owner->aio_bounce_buffers, ioh_node); +} diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a5b12b48f99..dc52d6165d4 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2104,6 +2104,7 @@ Permutation PermutationStep PermutationStepBlocker PermutationStepBlockerType +PgAioBounceBuffer PgAioCtl PgAioHandle PgAioHandleFlags -- 2.45.2.746.g06e570c0df.dirty --oy2jwuii6tssbict Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0015-bufmgr-Implement-AIO-write-support.patch"