pgbackrest.git / summary / log / commit / refs
commit dcddf3a58bc6c4a8fbc9ac2c000cd5ff7b669cb5
Author: David Steele <david@pgmasters.net>
Date: Thu Mar 19 17:16:05 2020 +0000
Limit backup file copy size to size reported at backup start.
If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.
This also reduces the likelihood of seeing torn pages during the copy. Torn pages can still occur in the middle of the file, though, so they must be handled.
doc/xml/release.xml | 10 ++++++++++
src/command/backup/file.c | 14 ++++++++++----
test/src/module/command/backupTest.c | 27 ++++++++++++++++-----------
3 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index a3cf1045f..681c8dca7 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -65,6 +65,16 @@
<p>These commands (e.g. <cmd>restore</cmd>, <cmd>archive-get</cmd>) never used the compress options but allowed them to be passed on the command line. Now they will error when these options are passed on the command line. If these errors occur then remove the unused options.</p>
</release-item>
+
+ <release-item>
+ <release-item-contributor-list>
+ <release-item-reviewer id="cynthia.shang"/>
+ </release-item-contributor-list>
+
+ <p>Limit backup file copy size to size reported at backup start.</p>
+
+ <p>If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.</p>
+ </release-item>
</release-improvement-list>
<release-development-list>
diff --git a/src/command/backup/file.c b/src/command/backup/file.c
index bd07a6a6f..5b814f8ca 100644
--- a/src/command/backup/file.c
+++ b/src/command/backup/file.c
@@ -84,8 +84,11 @@ backupFile(
// recopy.
if (delta)
{
- // Generate checksum/size for the pg file
- IoRead *read = storageReadIo(storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing));
+ // Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has grown
+ // since the manifest was built we don't need to consider the extra bytes since they will be replayed from WAL
+ // during recovery.
+ IoRead *read = storageReadIo(
+ storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .limit = VARUINT64(pgFileSize)));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
@@ -190,9 +193,12 @@ backupFile(
// Is the file compressible during the copy?
bool compressible = repoFileCompressType == compressTypeNone && cipherType == cipherTypeNone;
- // Setup pg file for read
+ // Setup pg file for read. Only read as many bytes as passed in pgFileSize. If the file is growing it does no good to
+ // copy data past the end of the size recorded in the manifest since those blocks will need to be replayed from WAL
+ // during recovery.
StorageRead *read = storageNewReadP(
- storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible);
+ storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible,
+ .limit = VARUINT64(pgFileSize));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c
index 5b36f125f..1b5ac52f4 100644
--- a/test/src/module/command/backupTest.c
+++ b/test/src/module/command/backupTest.c
@@ -562,6 +562,11 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
// Test pagechecksum
+
+ // Increase the file size but most of the following tests will still treat the file as size 9. This tests the common case
+ // where a file grows while a backup is running.
+ storagePutP(storageNewWriteP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile!!!"));
+
TEST_ASSIGN(
result,
backupFile(
@@ -665,13 +670,14 @@ testRun(void)
TEST_ASSIGN(
result,
backupFile(
- pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel,
- true, cipherTypeNone, NULL),
+ pgFile, false, 9999999, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1,
+ backupLabel, true, cipherTypeNone, NULL),
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
- TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
+ TEST_RESULT_UINT(result.copySize + result.repoSize, 24, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
+ TEST_RESULT_STR_Z(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69", "TEST");
TEST_RESULT_BOOL(
- (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
+ (strEqZ(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69") &&
storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy");
@@ -853,11 +859,11 @@ testRun(void)
pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
backupLabel, true, cipherTypeAes256Cbc, strNew("12345678")),
"pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
- TEST_RESULT_UINT(result.copySize, 9, " copy size set");
+ TEST_RESULT_UINT(result.copySize, 8, " copy size set");
TEST_RESULT_UINT(result.repoSize, 32, " repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
- (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
+ (strEqZ(result.copyChecksum, "acc972a8319d4903b839c64ec217faa3e77b4fcb") &&
storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy file (size missmatch) to encrypted repo success");
@@ -2126,8 +2132,8 @@ testRun(void)
"pg_data/backup_label {file, s=17}\n"
"pg_data/base {path}\n"
"pg_data/base/1 {path}\n"
- "pg_data/base/1/1 {file, s=4}\n"
- "pg_data/base/1/2 {file, s=4}\n"
+ "pg_data/base/1/1 {file, s=0}\n"
+ "pg_data/base/1/2 {file, s=2}\n"
"pg_data/base/1/3 {file, s=3}\n"
"pg_data/global {path}\n"
"pg_data/global/pg_control {file, s=8192}\n"
@@ -2143,9 +2149,8 @@ testRun(void)
",\"timestamp\":1571200000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1571200002}\n"
- "pg_data/base/1/1={\"checksum\":\"7110eda4d09e062aa5e4a390b0a572ac0d2c0220\",\"master\":false,\"size\":4"
- ",\"timestamp\":1571200000}\n"
- "pg_data/base/1/2={\"checksum\":\"2abd55e001c524cb2cf6300a89ca6366848a77d5\",\"master\":false,\"size\":4"
+ "pg_data/base/1/1={\"master\":false,\"size\":0,\"timestamp\":1571200000}\n"
+ "pg_data/base/1/2={\"checksum\":\"54ceb91256e8190e474aa752a6e0650a2df5ba37\",\"master\":false,\"size\":2"
",\"timestamp\":1571200000}\n"
"pg_data/base/1/3={\"checksum\":\"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8\",\"master\":false,\"size\":3"
",\"timestamp\":1571200000}\n"
[parent: e5bcc0c47e9d]