pgbackrest.git / summary / log / commit / refs
commit 5bdaa35fa5fea30bf7cdf10a5077cb3b661aa3db
Author: David Steele <david@pgmasters.net>
Date: Fri Sep 07 15:00:18 2018 +0000
Migrate walIsPartial(), walIsSegment(), and walSegmentFind() from Perl to C.
Also refactor regular expression defines to make them more reusable.
doc/xml/release.xml | 4 ++
src/Makefile | 2 +-
src/command/archive/common.c | 98 ++++++++++++++++++++++++++++++++++++
src/command/archive/common.h | 17 ++++++-
test/define.yaml | 2 +-
test/src/module/archive/commonTest.c | 61 ++++++++++++++++++++++
6 files changed, 180 insertions(+), 4 deletions(-)
diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 9853f50fd..3492e9357 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -38,6 +38,10 @@
<release-item>
<p>Add helper for repository storage.</p>
</release-item>
+
+ <release-item>
+ <p>Migrate <code>walIsPartial()</code>, <code>walIsSegment()</code>, and <code>walSegmentFind()</code> from Perl to C.</p>
+ </release-item>
</release-development-list>
</release-core-list>
</release>
diff --git a/src/Makefile b/src/Makefile
index b6210862e..cc58b1e93 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -140,7 +140,7 @@ install: pgbackrest
####################################################################################################################################
# Compile rules
####################################################################################################################################
-command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
+command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o
command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
diff --git a/src/command/archive/common.c b/src/command/archive/common.c
index 5e481d6a1..171d3ce69 100644
--- a/src/command/archive/common.c
+++ b/src/command/archive/common.c
@@ -10,8 +10,10 @@ Archive Push Command
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
+#include "common/regExp.h"
#include "common/wait.h"
#include "postgres/version.h"
+#include "protocol/storage/helper.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
@@ -111,6 +113,102 @@ archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confe
FUNCTION_DEBUG_RESULT(BOOL, result);
}
+/***********************************************************************************************************************************
+Is the segment partial?
+***********************************************************************************************************************************/
+bool
+walIsPartial(const String *walSegment)
+{
+ FUNCTION_DEBUG_BEGIN(logLevelTrace);
+ FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+ FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+ FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
+ FUNCTION_DEBUG_END();
+
+ FUNCTION_DEBUG_RESULT(BOOL, strEndsWithZ(walSegment, WAL_SEGMENT_PARTIAL_EXT));
+}
+
+/***********************************************************************************************************************************
+Is the file a segment or some other file (e.g. .history, .backup, etc)
+***********************************************************************************************************************************/
+bool
+walIsSegment(const String *walSegment)
+{
+ FUNCTION_DEBUG_BEGIN(logLevelTrace);
+ FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+ FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+ FUNCTION_DEBUG_END();
+
+ // Create the regular expression to identify WAL segments if it does not already exist
+ static RegExp *regExpSegment = NULL;
+
+ if (regExpSegment == NULL)
+ {
+ MEM_CONTEXT_BEGIN(memContextTop())
+ {
+ regExpSegment = regExpNew(strNew(WAL_SEGMENT_PARTIAL_REGEXP));
+ }
+ MEM_CONTEXT_END();
+ }
+
+ FUNCTION_DEBUG_RESULT(BOOL, regExpMatch(regExpSegment, walSegment));
+}
+
+/***********************************************************************************************************************************
+Find a WAL segment in the repository
+
+The file name can have several things appended such as a hash, compression extension, and partial extension so it is possible to
+have multiple files that match the segment, though more than one match is not a good thing.
+***********************************************************************************************************************************/
+String *
+walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment)
+{
+ FUNCTION_DEBUG_BEGIN(logLevelDebug);
+ FUNCTION_DEBUG_PARAM(STORAGE, storage);
+ FUNCTION_DEBUG_PARAM(STRING, archiveId);
+ FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+ FUNCTION_DEBUG_ASSERT(storage != NULL);
+ FUNCTION_DEBUG_ASSERT(archiveId != NULL);
+ FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+ FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
+ FUNCTION_DEBUG_END();
+
+ String *result = NULL;
+
+ MEM_CONTEXT_TEMP_BEGIN()
+ {
+ // Get a list of all WAL segments that match
+ StringList *list = storageListP(
+ storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))),
+ .expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)),
+ walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""));
+
+ // If there are results
+ if (list != NULL && strLstSize(list) > 0)
+ {
+ // Error if there is more than one match
+ if (strLstSize(list) > 1)
+ {
+ THROW_FMT(
+ ArchiveDuplicateError,
+ "duplicates found in archive for WAL segment %s: %s\n"
+ "HINT: are multiple primaries archiving to this stanza?",
+ strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", ")));
+ }
+
+ // Copy file name of WAL segment found into the calling context
+ memContextSwitch(MEM_CONTEXT_OLD());
+ result = strDup(strLstGet(list, 0));
+ }
+ }
+ MEM_CONTEXT_TEMP_END();
+
+ FUNCTION_DEBUG_RESULT(STRING, result);
+}
+
/***********************************************************************************************************************************
Get the next WAL segment given a WAL segment and WAL segment size
***********************************************************************************************************************************/
diff --git a/src/command/archive/common.h b/src/command/archive/common.h
index fea2a4cfd..8aab50eba 100644
--- a/src/command/archive/common.h
+++ b/src/command/archive/common.h
@@ -18,12 +18,22 @@ typedef enum
} ArchiveMode;
#include "common/type/stringList.h"
+#include "storage/storage.h"
/***********************************************************************************************************************************
WAL segment constants
***********************************************************************************************************************************/
-// Only match on a WAL segment without checksum appended
-#define WAL_SEGMENT_REGEXP "^[0-F]{24}$"
+// Extension for partial segments
+#define WAL_SEGMENT_PARTIAL_EXT ".partial"
+
+// Match when the first 24 characters match a WAL segment
+#define WAL_SEGMENT_PREFIX_REGEXP "^[0-F]{24}"
+
+// Match on a WAL segment without checksum appended
+#define WAL_SEGMENT_REGEXP WAL_SEGMENT_PREFIX_REGEXP "$"
+
+// Match on a WAL segment with partial allowed
+#define WAL_SEGMENT_PARTIAL_REGEXP WAL_SEGMENT_PREFIX_REGEXP "(\\.partial){0,1}$"
// Defines the size of standard WAL segment name -- this should never changed
#define WAL_SEGMENT_NAME_SIZE ((unsigned int)24)
@@ -36,6 +46,9 @@ Functions
***********************************************************************************************************************************/
bool archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confessOnError);
+bool walIsPartial(const String *walSegment);
+bool walIsSegment(const String *walSegment);
+String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment);
String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion);
StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range);
diff --git a/test/define.yaml b/test/define.yaml
index 2cb3ba288..64bf89d7e 100644
--- a/test/define.yaml
+++ b/test/define.yaml
@@ -593,7 +593,7 @@ unit:
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: common
- total: 3
+ total: 6
coverage:
command/archive/common: full
diff --git a/test/src/module/archive/commonTest.c b/test/src/module/archive/commonTest.c
index 60ec8f056..c6eb08b1c 100644
--- a/test/src/module/archive/commonTest.c
+++ b/test/src/module/archive/commonTest.c
@@ -5,6 +5,8 @@ Test Archive Common
#include "common/harnessConfig.h"
+#include "protocol/storage/helper.h"
+
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@@ -13,6 +15,9 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
+ // Create default storage object for testing
+ Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
+
// *****************************************************************************************************************************
if (testBegin("archiveAsyncStatus()"))
{
@@ -98,6 +103,62 @@ testRun(void)
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment)))));
}
+ // *****************************************************************************************************************************
+ if (testBegin("walIsPartial()"))
+ {
+ TEST_RESULT_BOOL(walIsPartial(strNew("000000010000000100000001")), false, "not partial");
+ TEST_RESULT_BOOL(walIsPartial(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial");
+ }
+
+ // *****************************************************************************************************************************
+ if (testBegin("walIsSegment()"))
+ {
+ TEST_RESULT_BOOL(walIsSegment(strNew("000000010000000100000001")), true, "wal segment");
+ TEST_RESULT_BOOL(walIsSegment(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial wal segment");
+ TEST_RESULT_BOOL(walIsSegment(strNew("0000001A.history")), false, "history file");
+ }
+
+ // *****************************************************************************************************************************
+ if (testBegin("walSegmentFind()"))
+ {
+ // Load configuration to set repo-path and stanza
+ StringList *argList = strLstNew();
+ strLstAddZ(argList, "pgbackrest");
+ strLstAddZ(argList, "--stanza=db");
+ strLstAdd(argList, strNewFmt("--repo-path=%s", testPath()));
+ strLstAddZ(argList, "archive-get");
+ harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+ TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path");
+
+ storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678"));
+ TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment");
+
+ storagePutNP(
+ storageNewWriteNP(
+ storageTest,
+ strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
+ NULL);
+
+ TEST_RESULT_STR(
+ strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))),
+ "123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
+
+ storagePutNP(
+ storageNewWriteNP(
+ storageTest,
+ strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz")),
+ NULL);
+
+ TEST_ERROR(
+ walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")),
+ ArchiveDuplicateError,
+ "duplicates found in archive for WAL segment 123456781234567812345678:"
+ " 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ ", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz"
+ "\nHINT: are multiple primaries archiving to this stanza?");
+ }
+
// *****************************************************************************************************************************
if (testBegin("walSegmentNext()"))
{
[parent: 96600760930d]