pg.ddx.io  pgbackrest/pgbackrest GitHub issues and pull requests  
help / color / mirror / Atom feed
From: dwsteele (@dwsteele) <noreply+dwsteele@github.com>
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
Subject: [pgbackrest/pgbackrest] PR #2927: Continue async archive-push to healthy repos after a repo error.
Date: Wed, 18 Nov 2026 13:50:53 +0000
Message-ID: <gh-pgbackrest-pgbackrest-2927@github.com> (raw)

Since a0c143593 async archive-push stopped scheduling new jobs after any job error, including a push that succeeded on some repositories but failed on others. Each run restarts at the oldest WAL segment that has not been acknowledged so the healthy repositories received no more WAL until the failed repository recovered.

Remove a repository from the run after it errors and continue pushing to the remaining repositories. Later segments report the repository as skipped after the error on the earlier segment. The run stops when no repositories remain or when a job errors outright, such as a missing WAL segment.

archivePushFile() returns repository errors instead of throwing them so the async process can determine which repositories failed. The sync process throws the same error as before.


inline patch 1: src/command/archive/push/file.c, src/command/archive/push/file.h, src/command/archive/push/protocol.c and 2 more (+210/-48 lines) download inline-1.patchdiff --git a/src/command/archive/push/file.c b/src/command/archive/push/file.c index 104b12e6ac..dec1b4d2d3 100644 --- a/src/command/archive/push/file.c +++ b/src/command/archive/push/file.c @@ -32,14 +32,25 @@ typedef enum // Helper to add errors to the list static void -archivePushErrorAdd(StringList *const errorList, const unsigned int repoIdx) +archivePushErrorAdd(List *const repoErrorList, const unsigned int repoIdx) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING_LIST, errorList); + FUNCTION_TEST_PARAM(LIST, repoErrorList); FUNCTION_TEST_PARAM(UINT, repoIdx); FUNCTION_TEST_END(); - strLstAddFmt(errorList, "%s: [%s] %s", cfgOptionGroupName(cfgOptGrpRepo, repoIdx), errorTypeName(errorType()), errorMessage()); + MEM_CONTEXT_OBJ_BEGIN(repoErrorList) + { + const ArchivePushFileRepoError repoError = + { + .repoIdx = repoIdx, + .message = strNewFmt( + "%s: [%s] %s", cfgOptionGroupName(cfgOptGrpRepo, repoIdx), errorTypeName(errorType()), errorMessage()), + }; + + lstAdd(repoErrorList, &repoError); + } + MEM_CONTEXT_OBJ_END(); FUNCTION_TEST_RETURN_VOID(); } @@ -47,18 +58,18 @@ archivePushErrorAdd(StringList *const errorList, const unsigned int repoIdx) static bool archivePushFileIo( const ArchivePushFileIoType type, IoWrite *const write, const Buffer *const buffer, const unsigned int repoIdx, - StringList *const errorList) + List *const repoErrorList) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_PARAM(IO_WRITE, write); FUNCTION_TEST_PARAM(BUFFER, buffer); FUNCTION_TEST_PARAM(UINT, repoIdx); - FUNCTION_TEST_PARAM(STRING_LIST, errorList); + FUNCTION_TEST_PARAM(LIST, repoErrorList); FUNCTION_TEST_END(); ASSERT(write != NULL); - ASSERT(errorList != NULL); + ASSERT(repoErrorList != NULL); bool result = true; @@ -84,7 +95,7 @@ archivePushFileIo( // Handle errors CATCH_ANY() { - archivePushErrorAdd(errorList, repoIdx); + archivePushErrorAdd(repoErrorList, repoIdx); result = false; } TRY_END(); @@ -120,11 +131,11 @@ archivePushFile( ASSERT(priorErrorList != NULL); ASSERT(lstSize(repoList) > 0); - ArchivePushFileResult result = {.warnList = strLstNew()}; + ArchivePushFileResult result = {.warnList = strLstNew(), .repoErrorList = lstNewP(sizeof(ArchivePushFileRepoError))}; MEM_CONTEXT_TEMP_BEGIN() { - StringList *const errorList = strLstDup(priorErrorList); + List *const repoErrorList = result.repoErrorList; // Is this a WAL segment? const bool isSegment = walIsSegment(archiveFile); @@ -182,7 +193,7 @@ archivePushFile( } CATCH_ANY() { - archivePushErrorAdd(errorList, repoData->repoIdx); + archivePushErrorAdd(repoErrorList, repoData->repoIdx); destinationCopy[repoListIdx] = false; } TRY_END(); @@ -283,7 +294,7 @@ archivePushFile( if (destinationCopy[repoListIdx]) { destinationCopy[repoListIdx] = archivePushFileIo( - archivePushFileIoTypeOpen, storageWriteIo(destination[repoListIdx]), NULL, repoIdx, errorList); + archivePushFileIoTypeOpen, storageWriteIo(destination[repoListIdx]), NULL, repoIdx, repoErrorList); } } @@ -303,7 +314,7 @@ archivePushFile( if (destinationCopy[repoListIdx]) { destinationCopy[repoListIdx] = archivePushFileIo( - archivePushFileIoTypeWrite, storageWriteIo(destination[repoListIdx]), read, repoIdx, errorList); + archivePushFileIoTypeWrite, storageWriteIo(destination[repoListIdx]), read, repoIdx, repoErrorList); } } @@ -322,15 +333,26 @@ archivePushFile( if (destinationCopy[repoListIdx]) { destinationCopy[repoListIdx] = archivePushFileIo( - archivePushFileIoTypeClose, storageWriteIo(destination[repoListIdx]), NULL, repoIdx, errorList); + archivePushFileIoTypeClose, storageWriteIo(destination[repoListIdx]), NULL, repoIdx, repoErrorList); } } } - // Throw any errors, even if some pushes were successful. It is important that PostgreSQL receives an error so it does not - // remove the file. - if (strLstSize(errorList) > 0) - THROW_FMT(CommandError, CFGCMD_ARCHIVE_PUSH " command encountered error(s):\n%s", strZ(strLstJoin(errorList, "\n"))); + // Return prior errors and repo errors, even if some pushes were successful + if (!strLstEmpty(priorErrorList) || !lstEmpty(repoErrorList)) + { + StringList *const errorList = strLstDup(priorErrorList); + + for (unsigned int repoErrorIdx = 0; repoErrorIdx < lstSize(repoErrorList); repoErrorIdx++) + strLstAdd(errorList, ((ArchivePushFileRepoError *)lstGet(repoErrorList, repoErrorIdx))->message); + + MEM_CONTEXT_PRIOR_BEGIN() + { + result.error = strNewFmt( + CFGCMD_ARCHIVE_PUSH " command encountered error(s):\n%s", strZ(strLstJoin(errorList, "\n"))); + } + MEM_CONTEXT_PRIOR_END(); + } } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/archive/push/file.h b/src/command/archive/push/file.h index d1ee1f454f..bdcbf9fd09 100644 --- a/src/command/archive/push/file.h +++ b/src/command/archive/push/file.h @@ -25,9 +25,17 @@ typedef struct ArchivePushFileRepoData /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ +typedef struct ArchivePushFileRepoError +{ + unsigned int repoIdx; // Repo index + const String *message; // Error message +} ArchivePushFileRepoError; + typedef struct ArchivePushFileResult { StringList *warnList; // Warnings from a successful operation + String *error; // Prior errors and repo errors, NULL when there are none + List *repoErrorList; // Errors from repos that failed (ArchivePushFileRepoError) } ArchivePushFileResult; // Copy a file from the source to the archive diff --git a/src/command/archive/push/protocol.c b/src/command/archive/push/protocol.c index 4a149d9b40..c41ad286ff 100644 --- a/src/command/archive/push/protocol.c +++ b/src/command/archive/push/protocol.c @@ -64,7 +64,20 @@ archivePushFileProtocol(PackRead *const param) priorErrorList); // Return result - pckWriteStrLstP(protocolServerResultData(result), fileResult.warnList); + PackWrite *const resultPack = protocolServerResultData(result); + + pckWriteStrLstP(resultPack, fileResult.warnList); + pckWriteStrP(resultPack, fileResult.error); + pckWriteArrayBeginP(resultPack); + + for (unsigned int repoErrorIdx = 0; repoErrorIdx < lstSize(fileResult.repoErrorList); repoErrorIdx++) + { + const ArchivePushFileRepoError *const repoError = lstGet(fileResult.repoErrorList, repoErrorIdx); + + pckWriteU32P(resultPack, repoError->repoIdx, .defaultWrite = true); + } + + pckWriteArrayEndP(resultPack); } MEM_CONTEXT_TEMP_END(); diff --git a/src/command/archive/push/push.c b/src/command/archive/push/push.c index c89d2dd3a1..36fd261f4e 100644 --- a/src/command/archive/push/push.c +++ b/src/command/archive/push/push.c @@ -420,6 +420,11 @@ cmdArchivePush(void) archiveInfo.pgSystemId, archiveFile, compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel), archiveInfo.repoList, archiveInfo.errorList); + // Throw any errors, even if some pushes were successful. It is important that PostgreSQL receives an error so it + // does not remove the file. + if (fileResult.error != NULL) + THROW(CommandError, strZ(fileResult.error)); + // If a warning was returned then log it for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileResult.warnList); warnIdx++) LOG_WARN(strZ(strLstGet(fileResult.warnList, warnIdx))); @@ -443,7 +448,7 @@ typedef struct ArchivePushAsyncData CompressType compressType; // Type of compression for WAL segments int compressLevel; // Compression level for wal files ArchivePushCheckResult archiveInfo; // Archive info - bool errorFound; // Has a job errored? If so, stop scheduling new jobs + bool errorFound; // Stop scheduling jobs when a job errored or no repos remain } ArchivePushAsyncData; static ProtocolParallelJob * @@ -461,9 +466,10 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx) // No special logic based on the client, we'll just get the next job (void)clientIdx; - // Get a new job if there are any left. Stop scheduling new jobs as soon as one errors so the async process exits and the - // next run rechecks the queue. Continuing past an error does not relieve disk pressure since PostgreSQL cannot recycle any - // WAL past the oldest unarchived segment, so the sooner the queue is rechecked the sooner WAL can be dropped if needed. + // Get a new job if there are any left. Stop scheduling new jobs as soon as a job errors or no repos remain so the async + // process exits and the next run rechecks the queue. Continuing past an error does not relieve disk pressure since + // PostgreSQL cannot recycle any WAL past the oldest unarchived segment, so the sooner the queue is rechecked the sooner WAL + // can be dropped if needed. ArchivePushAsyncData *const jobData = data; if (!jobData->errorFound && jobData->walFileIdx < strLstSize(jobData->walFileList)) @@ -513,6 +519,45 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx) FUNCTION_TEST_RETURN(PROTOCOL_PARALLEL_JOB, result); } +// Remove repos that errored from the repo list and add an error for each to the error list +static void +archivePushAsyncRepoRemove(ArchivePushCheckResult *const archiveInfo, PackRead *const jobResult, const String *const walFile) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, archiveInfo); + FUNCTION_TEST_PARAM(PACK_READ, jobResult); + FUNCTION_TEST_PARAM(STRING, walFile); + FUNCTION_TEST_END(); + + MEM_CONTEXT_TEMP_BEGIN() + { + pckReadArrayBeginP(jobResult); + + while (!pckReadNullP(jobResult)) + { + const unsigned int repoIdx = pckReadU32P(jobResult); + + // Remove the repo when it is in the repo list + for (unsigned int repoListIdx = 0; repoListIdx < lstSize(archiveInfo->repoList); repoListIdx++) + { + if (((const ArchivePushFileRepoData *)lstGet(archiveInfo->repoList, repoListIdx))->repoIdx == repoIdx) + { + lstRemoveIdx(archiveInfo->repoList, repoListIdx); + strLstAddFmt( + archiveInfo->errorList, "%s: skipped after error pushing WAL file '%s'", + cfgOptionGroupName(cfgOptGrpRepo, repoIdx), strZ(walFile)); + break; + } + } + } + + pckReadArrayEndP(jobResult); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + FN_EXTERN void cmdArchivePushAsync(void) { @@ -639,36 +684,56 @@ cmdArchivePushAsync(void) ProtocolParallelJob *const job = protocolParallelResult(parallelExec); const unsigned int processId = protocolParallelJobProcessId(job); const String *const walFile = varStr(protocolParallelJobKey(job)); + int errorCode = protocolParallelJobErrorCode(job); + const String *errorMessage = protocolParallelJobErrorMessage(job); // The job was successful - if (protocolParallelJobErrorCode(job) == 0) + if (errorCode == 0) { - // Output file warnings - const StringList *const fileWarnList = pckReadStrLstP(protocolParallelJobResult(job)); - - for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++) - LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx))); - - // Log success - LOG_DETAIL_PID_FMT(processId, "pushed WAL file '%s' to the archive", strZ(walFile)); - - // Write the status file - archiveAsyncStatusOkWrite( - archiveModePush, walFile, strLstEmpty(fileWarnList) ? NULL : strLstJoin(fileWarnList, "\n")); + PackRead *const jobResult = protocolParallelJobResult(job); + const StringList *const fileWarnList = pckReadStrLstP(jobResult); + const String *const fileError = pckReadStrP(jobResult); + + // The WAL file was pushed to all repos + if (fileError == NULL) + { + // Output file warnings + for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++) + LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx))); + + // Log success + LOG_DETAIL_PID_FMT(processId, "pushed WAL file '%s' to the archive", strZ(walFile)); + + // Write the status file + archiveAsyncStatusOkWrite( + archiveModePush, walFile, + strLstEmpty(fileWarnList) ? NULL : strLstJoin(fileWarnList, "\n")); + } + // Else one or more repos errored + else + { + archivePushAsyncRepoRemove(&jobData.archiveInfo, jobResult, walFile); + + if (lstEmpty(jobData.archiveInfo.repoList)) + jobData.errorFound = true; + + errorCode = errorTypeCode(&CommandError); + errorMessage = fileError; + } } // Else the job errored else - { jobData.errorFound = true; + // Log the error and write the status file + if (errorCode != 0) + { LOG_WARN_PID_FMT( processId, "could not push WAL file '%s' to the archive (will be retried): [%d] %s", strZ(walFile), - protocolParallelJobErrorCode(job), strZ(protocolParallelJobErrorMessage(job))); + errorCode, strZ(errorMessage)); - archiveAsyncStatusErrorWrite( - archiveModePush, walFile, protocolParallelJobErrorCode(job), - protocolParallelJobErrorMessage(job)); + archiveAsyncStatusErrorWrite(archiveModePush, walFile, errorCode, errorMessage); } protocolParallelJobFree(job); diff --git a/test/src/module/command/archive/pushTest.c b/test/src/module/command/archive/pushTest.c index ebe482a376..2d754efb5f 100644 --- a/test/src/module/command/archive/pushTest.c +++ b/test/src/module/command/archive/pushTest.c @@ -1268,29 +1268,31 @@ testRun(void) HRN_STORAGE_PUT_EMPTY(storagePgWrite(), zNewFmt("pg_wal/archive_status/00000001000000010000000%u.ready", walIdx)); } - // Make the repo archive path read-only so each push errors when it writes, simulating an unavailable repository + // Make the repo archive paths read-only so each push errors when it writes, simulating unavailable repositories HRN_STORAGE_MODE(storageTest, "repo/archive/test/18-1/0000000100000001", .mode = 0500); + HRN_STORAGE_MODE(storageTest, "repo3/archive/test/18-1/0000000100000001", .mode = 0500); argListTemp = strLstDup(argList); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync); - // The job that errors and the one already in flight (process-max is 1) both complete, then scheduling stops + // The job that errors on all repos and the one already in flight (process-max is 1) both complete, then scheduling stops TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); TEST_RESULT_LOG_FMT( "P00 INFO: push 6 WAL file(s) to archive: 000000010000000100000004...000000010000000100000009\n" "P01 WARN: could not push WAL file '000000010000000100000004' to the archive (will be retried): " - "[104] raised from local-1 shim protocol: archive-push command encountered error(s):\n" + "[104] archive-push command encountered error(s):\n" " repo1: [FileOpenError] unable to open file '" TEST_PATH "/repo/archive/test/18-1/0000000100000001/000000010000000100000004-%s' for write: [13] Permission denied\n" + " repo3: [FileOpenError] unable to open file '" TEST_PATH + "/repo3/archive/test/18-1/0000000100000001/000000010000000100000004-%s' for write: [13] Permission denied\n" "P01 WARN: could not push WAL file '000000010000000100000005' to the archive (will be retried): " - "[104] raised from local-1 shim protocol: archive-push command encountered error(s):\n" + "[104] archive-push command encountered error(s):\n" " repo1: [FileOpenError] unable to open file '" TEST_PATH "/repo/archive/test/18-1/0000000100000001/000000010000000100000005-%s' for write: [13] Permission denied\n" + " repo3: [FileOpenError] unable to open file '" TEST_PATH + "/repo3/archive/test/18-1/0000000100000001/000000010000000100000005-%s' for write: [13] Permission denied\n" "P00 WARN: stopped archive-push after an error, remaining WAL will be processed on the next run", - walBufferBatchSha1, walBufferBatchSha1); - - // Restore the repo mode so later tests can write - HRN_STORAGE_MODE(storageTest, "repo/archive/test/18-1/0000000100000001"); + walBufferBatchSha1, walBufferBatchSha1, walBufferBatchSha1, walBufferBatchSha1); // Only the segments processed before stopping have status files; the rest are left for the next run TEST_STORAGE_LIST( @@ -1299,6 +1301,58 @@ testRun(void) "000000010000000100000005.error\n", .comment = "check status files"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("push to remaining repos after a repo errors"); + + HRN_STORAGE_MODE(storageTest, "repo3/archive/test/18-1/0000000100000001"); + + // Later jobs push only to repo3 and report repo1 as skipped after the error on WAL 4 + TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); + TEST_RESULT_LOG_FMT( + "P00 INFO: push 6 WAL file(s) to archive: 000000010000000100000004...000000010000000100000009\n" + "P01 WARN: could not push WAL file '000000010000000100000004' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: [FileOpenError] unable to open file '" TEST_PATH + "/repo/archive/test/18-1/0000000100000001/000000010000000100000004-%s' for write: [13] Permission denied\n" + "P01 WARN: could not push WAL file '000000010000000100000005' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: [FileOpenError] unable to open file '" TEST_PATH + "/repo/archive/test/18-1/0000000100000001/000000010000000100000005-%s' for write: [13] Permission denied\n" + "P01 WARN: could not push WAL file '000000010000000100000006' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: skipped after error pushing WAL file '000000010000000100000004'\n" + "P01 WARN: could not push WAL file '000000010000000100000007' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: skipped after error pushing WAL file '000000010000000100000004'\n" + "P01 WARN: could not push WAL file '000000010000000100000008' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: skipped after error pushing WAL file '000000010000000100000004'\n" + "P01 WARN: could not push WAL file '000000010000000100000009' to the archive (will be retried): " + "[104] archive-push command encountered error(s):\n" + " repo1: skipped after error pushing WAL file '000000010000000100000004'", + walBufferBatchSha1, walBufferBatchSha1); + + for (unsigned int walIdx = 4; walIdx <= 9; walIdx++) + { + TEST_STORAGE_EXISTS( + storageTest, + zNewFmt("repo3/archive/test/18-1/0000000100000001/00000001000000010000000%u-%s", walIdx, walBufferBatchSha1), + .comment = "check repo3 for WAL file"); + } + + TEST_STORAGE_LIST( + storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, + "000000010000000100000004.error\n" + "000000010000000100000005.error\n" + "000000010000000100000006.error\n" + "000000010000000100000007.error\n" + "000000010000000100000008.error\n" + "000000010000000100000009.error\n", + .comment = "check status files"); + + // Restore the repo mode so later tests can write + HRN_STORAGE_MODE(storageTest, "repo/archive/test/18-1/0000000100000001"); + // Uninstall local command handler shim hrnProtocolLocalShimUninstall(); }

view thread (9+ messages)  latest in thread

Message-ID: <gh-pgbackrest-pgbackrest-2927@github.com>
Permalink:  ../gh-pgbackrest-pgbackrest-2927@github.com/

 ·  · 


This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox