pgbackrest.git / summary / log / commit / refs

commit    102ce5dee47e8d9a8d2eeaf2b2cc2bac105914ef
Author:   David Steele <david@pgmasters.net>
Commit:   GitHub <noreply@github.com>
Date:     Thu Oct 06 02:28:31 2022 +0000

    Add persistent reference list to manifest.
    
    The reference list was previously built at load time from whichever references existed in the file list. This was sufficient since the list was for informational purposes only.
    
    The block incremental feature will require a reference list that contains all prior backups, even those that are not explicitly referenced from the manifest. Therefore it makes sense to build and persist a manifest list rather than building it at load time.
    
    This list can still be used for informational purposes, though it needs to be sorted since the list it sill built for older manifest versions and may not be in sorted order.
    
    Add strLstFindIdx() to find references in the list.


doc/xml/release.xml | 4 +++ src/common/type/stringList.c | 21 ++++++++++++++++ src/common/type/stringList.h | 12 +++++++++ src/info/infoBackup.c | 9 +++---- src/info/manifest.c | 29 +++++++++++++++++++--- src/info/manifest.h | 8 ++++++ test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm | 5 ++++ test/src/common/harnessLog.c | 2 +- test/src/module/command/backupTest.c | 1 + test/src/module/command/restoreTest.c | 5 ++++ test/src/module/common/typeStringTest.c | 7 ++++++ test/src/module/info/manifestTest.c | 24 ++++++++++++++++-- 12 files changed, 114 insertions(+), 13 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 14b49b78c..cae0b126a 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -54,10 +54,14 @@ <commit subject="Add manifest flags for file processing during backup."> <github-pull-request id="1889"/> </commit> + <commit subject="Add persistent reference list to manifest."> + <github-pull-request id="1890"/> + </commit> <release-item-contributor-list> <release-item-contributor id="david.steele"/> <release-item-reviewer id="stefan.fercot"/> + <release-item-reviewer id="reid.thompson"/> <release-item-reviewer id="john.morris"/> </release-item-contributor-list> diff --git a/src/common/type/stringList.c b/src/common/type/stringList.c index 35991390d..aebf28464 100644 --- a/src/common/type/stringList.c +++ b/src/common/type/stringList.c @@ -274,6 +274,27 @@ strLstAddZSubN(StringList *const this, const char *const string, const size_t of FUNCTION_TEST_RETURN(STRING, result); } +/**********************************************************************************************************************************/ +unsigned int +strLstFindIdx(const StringList *const this, const String *const string, const StrLstFindIdxParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING_LIST, this); + FUNCTION_TEST_PARAM(STRING, string); + FUNCTION_TEST_PARAM(BOOL, param.required); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(string != NULL); + + const unsigned int result = lstFindIdx((List *)this, &string); + + if (result == LIST_NOT_FOUND && param.required) + THROW_FMT(AssertError, "unable to find '%s' in string list", strZ(string)); + + FUNCTION_TEST_RETURN(UINT, result); +} + /**********************************************************************************************************************************/ String * strLstInsert(StringList *this, unsigned int listIdx, const String *string) diff --git a/src/common/type/stringList.h b/src/common/type/stringList.h index 81e5bff0c..fd8da8fa4 100644 --- a/src/common/type/stringList.h +++ b/src/common/type/stringList.h @@ -95,6 +95,18 @@ strLstExists(const StringList *const this, const String *const string) return lstExists((List *)this, &string); } +// Find string index in the list +typedef struct StrLstFindIdxParam +{ + VAR_PARAM_HEADER; + bool required; +} StrLstFindIdxParam; + +#define strLstFindIdxP(this, string, ...) \ + strLstFindIdx(this, string, (StrLstFindIdxParam){VAR_PARAM_INIT, __VA_ARGS__}) + +unsigned int strLstFindIdx(const StringList *this, const String *string, StrLstFindIdxParam param); + // Insert into the list String *strLstInsert(StringList *this, unsigned int listIdx, const String *string); diff --git a/src/info/infoBackup.c b/src/info/infoBackup.c index f2c888020..a255a7751 100644 --- a/src/info/infoBackup.c +++ b/src/info/infoBackup.c @@ -391,7 +391,6 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest) uint64_t backupSizeDelta = 0; uint64_t backupRepoSize = 0; uint64_t backupRepoSizeDelta = 0; - StringList *referenceList = strLstNew(); bool backupError = false; for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) @@ -402,9 +401,7 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest) backupRepoSize += file.sizeRepo > 0 ? file.sizeRepo : file.size; // If a reference to a file exists, then it is in a previous backup and the delta calculation was already done - if (file.reference != NULL) - strLstAddIfMissing(referenceList, file.reference); - else + if (file.reference == NULL) { backupSizeDelta += file.size; backupRepoSizeDelta += file.sizeRepo > 0 ? file.sizeRepo : file.size; @@ -450,8 +447,8 @@ infoBackupDataAdd(const InfoBackup *this, const Manifest *manifest) if (manData->backupType != backupTypeFull) { - strLstSort(referenceList, sortOrderAsc); - infoBackupData.backupReference = strLstDup(referenceList); + // This list may not be sorted for manifests created before the reference list was added + infoBackupData.backupReference = strLstSort(strLstDup(manifestReferenceList(manifest)), sortOrderAsc); infoBackupData.backupPrior = strDup(manData->backupLabelPrior); } diff --git a/src/info/manifest.c b/src/info/manifest.c index 128cfc73b..7098306cf 100644 --- a/src/info/manifest.c +++ b/src/info/manifest.c @@ -34,7 +34,6 @@ struct Manifest { ManifestPub pub; // Publicly accessible variables StringList *ownerList; // List of users/groups - StringList *referenceList; // List of file references const String *fileUserDefault; // Default file user name const String *fileGroupDefault; // Default file group name @@ -186,7 +185,7 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file) if (file->reference != NULL) { cvtUInt64ToVarInt128( - (uintptr_t)strLstAddIfMissing(manifest->referenceList, file->reference), buffer, &bufferPos, sizeof(buffer)); + strLstFindIdxP(manifest->pub.referenceList, file->reference, .required = true), buffer, &bufferPos, sizeof(buffer)); } // Mode @@ -283,7 +282,10 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const // Reference if (flag & (1 << manifestFilePackFlagReference)) - result.reference = (const String *)(uintptr_t)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX); + { + result.reference = strLstGet( + manifest->pub.referenceList, (unsigned int)cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX)); + } // Mode if (flag & (1 << manifestFilePackFlagMode)) @@ -481,9 +483,9 @@ manifestNewInternal(void) .linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr), .pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr), .targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr), + .referenceList = strLstNew(), }, .ownerList = strLstNew(), - .referenceList = strLstNew(), }; FUNCTION_TEST_RETURN(MANIFEST, this); @@ -1426,6 +1428,9 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type // Set prior backup label this->pub.data.backupLabelPrior = strDup(manifestPrior->pub.data.backupLabel); + // Copy reference list + this->pub.referenceList = strLstDup(manifestPrior->pub.referenceList); + // Set diff/incr backup type this->pub.data.backupType = type; } @@ -1665,6 +1670,7 @@ manifestBuildComplete( #define MANIFEST_KEY_BACKUP_LSN_START "backup-lsn-start" #define MANIFEST_KEY_BACKUP_LSN_STOP "backup-lsn-stop" #define MANIFEST_KEY_BACKUP_PRIOR "backup-prior" +#define MANIFEST_KEY_BACKUP_REFERENCE "backup-reference" #define MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START "backup-timestamp-copy-start" #define MANIFEST_KEY_BACKUP_TIMESTAMP_START "backup-timestamp-start" #define MANIFEST_KEY_BACKUP_TIMESTAMP_STOP "backup-timestamp-stop" @@ -1720,6 +1726,7 @@ typedef struct ManifestLoadData { MemContext *memContext; // Mem context for data needed only during load Manifest *manifest; // Manifest info + bool referenceListFound; // Was a reference list found? List *linkFoundList; // Values found in links const Variant *linkGroupDefault; // Link default group @@ -1837,8 +1844,13 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri // Reference if (jsonReadKeyExpectStrId(json, MANIFEST_KEY_REFERENCE)) + { file.reference = jsonReadStr(json); + if (!loadData->referenceListFound) + file.reference = strLstAddIfMissing(manifest->pub.referenceList, file.reference); + } + // If "repo-size" is not present in the manifest file, then it is the same as size (i.e. uncompressed) - to save space, // the repo-size is only stored in the manifest file if it is different than size. const bool sizeRepoExists = jsonReadKeyExpectStrId(json, MANIFEST_KEY_SIZE_REPO); @@ -2059,6 +2071,11 @@ manifestLoadCallback(void *callbackData, const String *const section, const Stri manifest->pub.data.lsnStop = varStr(jsonToVar(value)); else if (strEqZ(key, MANIFEST_KEY_BACKUP_PRIOR)) manifest->pub.data.backupLabelPrior = varStr(jsonToVar(value)); + else if (strEqZ(key, MANIFEST_KEY_BACKUP_REFERENCE)) + { + manifest->pub.referenceList = strLstNewSplitZ(varStr(jsonToVar(value)), ","); + loadData->referenceListFound = true; + } else if (strEqZ(key, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START)) manifest->pub.data.backupTimestampCopyStart = (time_t)varUInt64(jsonToVar(value)); else if (strEqZ(key, MANIFEST_KEY_BACKUP_TIMESTAMP_START)) @@ -2305,6 +2322,9 @@ manifestSaveCallback(void *const callbackData, const String *const sectionNext, jsonFromVar(VARSTR(manifest->pub.data.backupLabelPrior))); } + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP, MANIFEST_KEY_BACKUP_REFERENCE, + jsonFromVar(VARSTR(strLstJoin(manifest->pub.referenceList, ",")))); infoSaveValue( infoSaveData, MANIFEST_SECTION_BACKUP, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START, jsonFromVar(VARINT64(manifest->pub.data.backupTimestampCopyStart))); @@ -3046,6 +3066,7 @@ manifestBackupLabelSet(Manifest *this, const String *backupLabel) MEM_CONTEXT_BEGIN(this->pub.memContext) { this->pub.data.backupLabel = strDup(backupLabel); + strLstAdd(this->pub.referenceList, backupLabel); } MEM_CONTEXT_END(); diff --git a/src/info/manifest.h b/src/info/manifest.h index f95029cee..8cb6710c5 100644 --- a/src/info/manifest.h +++ b/src/info/manifest.h @@ -182,6 +182,7 @@ typedef struct ManifestPub List *linkList; // List of links List *pathList; // List of paths List *targetList; // List of targets + StringList *referenceList; // List of file references } ManifestPub; // Get/set the cipher subpassphrase @@ -204,6 +205,13 @@ manifestData(const Manifest *const this) return &(THIS_PUB(Manifest)->data); } +// Get reference list +FN_INLINE_ALWAYS const StringList * +manifestReferenceList(const Manifest *const this) +{ + return THIS_PUB(Manifest)->referenceList; +} + // Set backup label void manifestBackupLabelSet(Manifest *this, const String *backupLabel); diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm index aafb04819..71786ecb8 100644 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm @@ -575,6 +575,8 @@ sub backupCompare my $oActualManifest = new pgBackRestTest::Env::Manifest( $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); + ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{'backup-reference'} = + $oActualManifest->get(MANIFEST_SECTION_BACKUP, 'backup-reference'); ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} = @@ -2122,6 +2124,9 @@ sub restoreCompare $oActualManifest->set( MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL}); + $oActualManifest->set( + MANIFEST_SECTION_BACKUP, 'backup-reference', undef, + $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-reference'}); $oActualManifest->set( MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START}); diff --git a/test/src/common/harnessLog.c b/test/src/common/harnessLog.c index a30a0996b..1cee2c27e 100644 --- a/test/src/common/harnessLog.c +++ b/test/src/common/harnessLog.c @@ -353,7 +353,7 @@ hrnLogReplace(void) if (logReplace->version) { - unsigned int index = lstFindIdx((List *)logReplace->matchList, &match); + unsigned int index = strLstFindIdxP(logReplace->matchList, match); if (index == LIST_NOT_FOUND) { diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index dad89f86d..2d2b142f7 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -2547,6 +2547,7 @@ testRun(void) const String *resumeLabel = backupLabelCreate( backupTypeDiff, manifestData(manifestPrior)->backupLabel, backupTimeStart); manifestBackupLabelSet(manifestResume, resumeLabel); + strLstAddZ(manifestResume->pub.referenceList, "BOGUS"); // Reference in manifest HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), zNewFmt(STORAGE_REPO_BACKUP "/%s/pg_data/PG_VERSION.gz", strZ(resumeLabel))); diff --git a/test/src/module/command/restoreTest.c b/test/src/module/command/restoreTest.c index fedf4ed46..3268ff7cb 100644 --- a/test/src/module/command/restoreTest.c +++ b/test/src/module/command/restoreTest.c @@ -2427,6 +2427,11 @@ testRun(void) manifest->pub.data.backupType = backupTypeIncr; manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this + manifest->pub.referenceList = strLstNew(); + strLstAddZ(manifest->pub.referenceList, TEST_LABEL_FULL); + strLstAddZ(manifest->pub.referenceList, TEST_LABEL_DIFF); + strLstAddZ(manifest->pub.referenceList, TEST_LABEL_INCR); + // Data directory manifestTargetAdd(manifest, &(ManifestTarget){.name = MANIFEST_TARGET_PGDATA_STR, .path = pgPath}); manifestPathAdd( diff --git a/test/src/module/common/typeStringTest.c b/test/src/module/common/typeStringTest.c index aed03b094..346f0d2a7 100644 --- a/test/src/module/common/typeStringTest.c +++ b/test/src/module/common/typeStringTest.c @@ -368,6 +368,13 @@ testRun(void) TEST_RESULT_UINT(strLstSize(list), 9, "list size"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("strLstFindIdxP()"); + + TEST_RESULT_UINT(strLstFindIdxP(list, STRDEF("STR05")), 5, "find STR05"); + TEST_RESULT_UINT(strLstFindIdxP(list, STRDEF("STR10")), LIST_NOT_FOUND, "find missing STR10"); + TEST_ERROR(strLstFindIdxP(list, STRDEF("STR10"), .required = true), AssertError, "unable to find 'STR10' in string list"); + // Read them back and check values // ------------------------------------------------------------------------------------------------------------------------- for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++) diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index 900a4bfce..849c42d1d 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -40,6 +40,7 @@ testRun(void) #define TEST_MANIFEST_HEADER \ "[backup]\n" \ "backup-label=null\n" \ + "backup-reference=\"\"\n" \ "backup-timestamp-copy-start=0\n" \ "backup-timestamp-start=0\n" \ "backup-timestamp-stop=0\n" \ @@ -49,6 +50,7 @@ testRun(void) "[backup]\n" \ "backup-bundle=true\n" \ "backup-label=null\n" \ + "backup-reference=\"\"\n" \ "backup-timestamp-copy-start=0\n" \ "backup-timestamp-start=0\n" \ "backup-timestamp-stop=0\n" \ @@ -1002,7 +1004,9 @@ testRun(void) #define TEST_MANIFEST_HEADER_PRE \ "[backup]\n" \ "backup-label=null\n" \ - "backup-prior=\"20190101-010101F\"\n" \ + "backup-prior=\"20190101-010101F\"\n" + + #define TEST_MANIFEST_HEADER_MID \ "backup-timestamp-copy-start=0\n" \ "backup-timestamp-start=0\n" \ "backup-timestamp-stop=0\n" \ @@ -1085,6 +1089,7 @@ testRun(void) { manifestPrior = manifestNewInternal(); manifestPrior->pub.data.backupLabel = strNewZ("20190101-010101F"); + strLstAdd(manifestPrior->pub.referenceList, manifestPrior->pub.data.backupLabel); manifestFileAdd( manifestPrior, @@ -1112,6 +1117,8 @@ testRun(void) strNewBuf(contentSave), strNewBuf(harnessInfoChecksumZ( TEST_MANIFEST_HEADER_PRE + "backup-reference=\"20190101-010101F\"\n" + TEST_MANIFEST_HEADER_MID "option-delta=false\n" TEST_MANIFEST_HEADER_POST "\n" @@ -1135,6 +1142,7 @@ testRun(void) TEST_TITLE("delta enabled before validation"); manifest->pub.data.backupOptionDelta = BOOL_TRUE_VAR; + strLstAddZ(manifestPrior->pub.referenceList, "20190101-010101F_20190202-010101D"); lstClear(manifest->pub.fileList); manifestFileAdd( manifest, @@ -1186,6 +1194,8 @@ testRun(void) strNewBuf(contentSave), strNewBuf(harnessInfoChecksumZ( TEST_MANIFEST_HEADER_PRE + "backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n" + TEST_MANIFEST_HEADER_MID "option-delta=true\n" TEST_MANIFEST_HEADER_POST "\n" @@ -1244,6 +1254,8 @@ testRun(void) strNewBuf(contentSave), strNewBuf(harnessInfoChecksumZ( TEST_MANIFEST_HEADER_PRE + "backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n" + TEST_MANIFEST_HEADER_MID "option-delta=true\n" TEST_MANIFEST_HEADER_POST "\n" @@ -1298,6 +1310,8 @@ testRun(void) strNewBuf(contentSave), strNewBuf(harnessInfoChecksumZ( TEST_MANIFEST_HEADER_PRE + "backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n" + TEST_MANIFEST_HEADER_MID "option-delta=true\n" TEST_MANIFEST_HEADER_POST "\n" @@ -1360,6 +1374,8 @@ testRun(void) strNewBuf(contentSave), strNewBuf(harnessInfoChecksumZ( TEST_MANIFEST_HEADER_PRE + "backup-reference=\"20190101-010101F,20190101-010101F_20190202-010101D\"\n" + TEST_MANIFEST_HEADER_MID "option-delta=true\n" "option-hardlink=false\n" "option-online=true\n" @@ -1377,6 +1393,7 @@ testRun(void) "check manifest"); #undef TEST_MANIFEST_HEADER_PRE + #undef TEST_MANIFEST_HEADER_MID #undef TEST_MANIFEST_HEADER_POST #undef TEST_MANIFEST_FILE_DEFAULT #undef TEST_MANIFEST_PATH_DEFAULT @@ -1392,6 +1409,7 @@ testRun(void) ( "[backup]\n" "backup-label=\"20190808-163540F\"\n" + "backup-reference=\"20190808-163540F\"\n" "backup-timestamp-copy-start=1565282141\n" "backup-timestamp-start=1565282140\n" "backup-timestamp-stop=1565282142\n" @@ -1419,7 +1437,8 @@ testRun(void) "cipher-pass=\"somepass\"\n" "\n" "[target:file]\n" - "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":4,\"timestamp\":1565282114}\n" + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"reference\":\"20190808-163540F\"" + ",\"size\":4,\"timestamp\":1565282114}\n" "\n" "[target:file:default]\n" "group=\"group1\"\n" @@ -1477,6 +1496,7 @@ testRun(void) "backup-lsn-start=\"285/89000028\"\n" \ "backup-lsn-stop=\"285/89001F88\"\n" \ "backup-prior=\"20190818-084502F\"\n" \ + "backup-reference=\"20190818-084502F_20190819-084506D,20190818-084502F,20190818-084502F_20190820-084502D\"\n" \ "backup-timestamp-copy-start=1565282141\n" \ "backup-timestamp-start=1565282140\n" \ "backup-timestamp-stop=1565282142\n" \ [parent: c647bcb50954]