pgbackrest.git / summary / log / commit / refs

commit    5bdaa35fa5fea30bf7cdf10a5077cb3b661aa3db
Author:   David Steele <david@pgmasters.net>
Date:     Fri Sep 07 15:00:18 2018 +0000

    Migrate walIsPartial(), walIsSegment(), and walSegmentFind() from Perl to C.
    
    Also refactor regular expression defines to make them more reusable.


doc/xml/release.xml | 4 ++ src/Makefile | 2 +- src/command/archive/common.c | 98 ++++++++++++++++++++++++++++++++++++ src/command/archive/common.h | 17 ++++++- test/define.yaml | 2 +- test/src/module/archive/commonTest.c | 61 ++++++++++++++++++++++ 6 files changed, 180 insertions(+), 4 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 9853f50fd..3492e9357 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -38,6 +38,10 @@ <release-item> <p>Add helper for repository storage.</p> </release-item> + + <release-item> + <p>Migrate <code>walIsPartial()</code>, <code>walIsSegment()</code>, and <code>walSegmentFind()</code> from Perl to C.</p> + </release-item> </release-development-list> </release-core-list> </release> diff --git a/src/Makefile b/src/Makefile index b6210862e..cc58b1e93 100644 --- a/src/Makefile +++ b/src/Makefile @@ -140,7 +140,7 @@ install: pgbackrest #################################################################################################################################### # Compile rules #################################################################################################################################### -command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h +command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h $(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h diff --git a/src/command/archive/common.c b/src/command/archive/common.c index 5e481d6a1..171d3ce69 100644 --- a/src/command/archive/common.c +++ b/src/command/archive/common.c @@ -10,8 +10,10 @@ Archive Push Command #include "common/debug.h" #include "common/log.h" #include "common/memContext.h" +#include "common/regExp.h" #include "common/wait.h" #include "postgres/version.h" +#include "protocol/storage/helper.h" #include "storage/helper.h" /*********************************************************************************************************************************** @@ -111,6 +113,102 @@ archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confe FUNCTION_DEBUG_RESULT(BOOL, result); } +/*********************************************************************************************************************************** +Is the segment partial? +***********************************************************************************************************************************/ +bool +walIsPartial(const String *walSegment) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STRING, walSegment); + + FUNCTION_DEBUG_ASSERT(walSegment != NULL); + FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment)); + FUNCTION_DEBUG_END(); + + FUNCTION_DEBUG_RESULT(BOOL, strEndsWithZ(walSegment, WAL_SEGMENT_PARTIAL_EXT)); +} + +/*********************************************************************************************************************************** +Is the file a segment or some other file (e.g. .history, .backup, etc) +***********************************************************************************************************************************/ +bool +walIsSegment(const String *walSegment) +{ + FUNCTION_DEBUG_BEGIN(logLevelTrace); + FUNCTION_DEBUG_PARAM(STRING, walSegment); + + FUNCTION_DEBUG_ASSERT(walSegment != NULL); + FUNCTION_DEBUG_END(); + + // Create the regular expression to identify WAL segments if it does not already exist + static RegExp *regExpSegment = NULL; + + if (regExpSegment == NULL) + { + MEM_CONTEXT_BEGIN(memContextTop()) + { + regExpSegment = regExpNew(strNew(WAL_SEGMENT_PARTIAL_REGEXP)); + } + MEM_CONTEXT_END(); + } + + FUNCTION_DEBUG_RESULT(BOOL, regExpMatch(regExpSegment, walSegment)); +} + +/*********************************************************************************************************************************** +Find a WAL segment in the repository + +The file name can have several things appended such as a hash, compression extension, and partial extension so it is possible to +have multiple files that match the segment, though more than one match is not a good thing. +***********************************************************************************************************************************/ +String * +walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment) +{ + FUNCTION_DEBUG_BEGIN(logLevelDebug); + FUNCTION_DEBUG_PARAM(STORAGE, storage); + FUNCTION_DEBUG_PARAM(STRING, archiveId); + FUNCTION_DEBUG_PARAM(STRING, walSegment); + + FUNCTION_DEBUG_ASSERT(storage != NULL); + FUNCTION_DEBUG_ASSERT(archiveId != NULL); + FUNCTION_DEBUG_ASSERT(walSegment != NULL); + FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment)); + FUNCTION_DEBUG_END(); + + String *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + // Get a list of all WAL segments that match + StringList *list = storageListP( + storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))), + .expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)), + walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : "")); + + // If there are results + if (list != NULL && strLstSize(list) > 0) + { + // Error if there is more than one match + if (strLstSize(list) > 1) + { + THROW_FMT( + ArchiveDuplicateError, + "duplicates found in archive for WAL segment %s: %s\n" + "HINT: are multiple primaries archiving to this stanza?", + strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", "))); + } + + // Copy file name of WAL segment found into the calling context + memContextSwitch(MEM_CONTEXT_OLD()); + result = strDup(strLstGet(list, 0)); + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_DEBUG_RESULT(STRING, result); +} + /*********************************************************************************************************************************** Get the next WAL segment given a WAL segment and WAL segment size ***********************************************************************************************************************************/ diff --git a/src/command/archive/common.h b/src/command/archive/common.h index fea2a4cfd..8aab50eba 100644 --- a/src/command/archive/common.h +++ b/src/command/archive/common.h @@ -18,12 +18,22 @@ typedef enum } ArchiveMode; #include "common/type/stringList.h" +#include "storage/storage.h" /*********************************************************************************************************************************** WAL segment constants ***********************************************************************************************************************************/ -// Only match on a WAL segment without checksum appended -#define WAL_SEGMENT_REGEXP "^[0-F]{24}$" +// Extension for partial segments +#define WAL_SEGMENT_PARTIAL_EXT ".partial" + +// Match when the first 24 characters match a WAL segment +#define WAL_SEGMENT_PREFIX_REGEXP "^[0-F]{24}" + +// Match on a WAL segment without checksum appended +#define WAL_SEGMENT_REGEXP WAL_SEGMENT_PREFIX_REGEXP "$" + +// Match on a WAL segment with partial allowed +#define WAL_SEGMENT_PARTIAL_REGEXP WAL_SEGMENT_PREFIX_REGEXP "(\\.partial){0,1}$" // Defines the size of standard WAL segment name -- this should never changed #define WAL_SEGMENT_NAME_SIZE ((unsigned int)24) @@ -36,6 +46,9 @@ Functions ***********************************************************************************************************************************/ bool archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confessOnError); +bool walIsPartial(const String *walSegment); +bool walIsSegment(const String *walSegment); +String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment); String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion); StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range); diff --git a/test/define.yaml b/test/define.yaml index 2cb3ba288..64bf89d7e 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -593,7 +593,7 @@ unit: test: # ---------------------------------------------------------------------------------------------------------------------------- - name: common - total: 3 + total: 6 coverage: command/archive/common: full diff --git a/test/src/module/archive/commonTest.c b/test/src/module/archive/commonTest.c index 60ec8f056..c6eb08b1c 100644 --- a/test/src/module/archive/commonTest.c +++ b/test/src/module/archive/commonTest.c @@ -5,6 +5,8 @@ Test Archive Common #include "common/harnessConfig.h" +#include "protocol/storage/helper.h" + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -13,6 +15,9 @@ testRun(void) { FUNCTION_HARNESS_VOID(); + // Create default storage object for testing + Storage *storageTest = storageNewP(strNew(testPath()), .write = true); + // ***************************************************************************************************************************** if (testBegin("archiveAsyncStatus()")) { @@ -98,6 +103,62 @@ testRun(void) unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))))); } + // ***************************************************************************************************************************** + if (testBegin("walIsPartial()")) + { + TEST_RESULT_BOOL(walIsPartial(strNew("000000010000000100000001")), false, "not partial"); + TEST_RESULT_BOOL(walIsPartial(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial"); + } + + // ***************************************************************************************************************************** + if (testBegin("walIsSegment()")) + { + TEST_RESULT_BOOL(walIsSegment(strNew("000000010000000100000001")), true, "wal segment"); + TEST_RESULT_BOOL(walIsSegment(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial wal segment"); + TEST_RESULT_BOOL(walIsSegment(strNew("0000001A.history")), false, "history file"); + } + + // ***************************************************************************************************************************** + if (testBegin("walSegmentFind()")) + { + // Load configuration to set repo-path and stanza + StringList *argList = strLstNew(); + strLstAddZ(argList, "pgbackrest"); + strLstAddZ(argList, "--stanza=db"); + strLstAdd(argList, strNewFmt("--repo-path=%s", testPath())); + strLstAddZ(argList, "archive-get"); + harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); + + TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path"); + + storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678")); + TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment"); + + storagePutNP( + storageNewWriteNP( + storageTest, + strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")), + NULL); + + TEST_RESULT_STR( + strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))), + "123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment"); + + storagePutNP( + storageNewWriteNP( + storageTest, + strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz")), + NULL); + + TEST_ERROR( + walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), + ArchiveDuplicateError, + "duplicates found in archive for WAL segment 123456781234567812345678:" + " 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz" + "\nHINT: are multiple primaries archiving to this stanza?"); + } + // ***************************************************************************************************************************** if (testBegin("walSegmentNext()")) { [parent: 96600760930d]