postgres.git / summary / log / commit / refs
commit 61aec3f3bc91c42cd08d1947794ddcc258020869
Author: Michael Paquier <michael@paquier.xyz>
Date: Tue Oct 27 04:53:59 2026 +0000
First released in: 15.20, 16.16, 17.12, 18.7, 19.0
Avoid truncated or zero-padded copies of replaced files in backups
Base backups call stat() on each file while scanning the data directory,
and later send exactly that many bytes. If the file is replaced by for
example a concurrent rename(), the backup contains the new file
truncated or zero-padded to the size of the old one. This is harmless
for relation files as WAL replay repairs them. However, it causes harm
for non-relation files manipulated as temporary file renamed in place.
Here is a list of non-relation files that use a temporary file followed
by a final rename() (I may be missing some):
* postgresql.auto.conf
* pg_logical/replorigin_checkpoint (a damaged copy fails its CRC check
and prevents startup)
* pg_prewarm's autoprewarm.blocks
* pg_stash_advice.tsv.
When sending files that are not relation files, fstat() is called again
on the opened file and the refreshed stats are used for the tar header
and the backup manifest. Since the rename
is atomic, the opened file is always complete. Relation files keep using
the earlier stat, to avoid copying data that WAL replay would restore
anyway (because that's cheaper).
Back-patch down to v15. v14 is going to be EOL soon, so I have decided
to leave it out of the picture (risk/reward thing).
Author: David Steele <david@pgbackrest.org>
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Reviewed-by: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Shihao Zhong <zhong950419@gmail.com>
Discussion: https://postgr.es/m/CAGRkXqS_On-pLgcv6xjH8mvGhdcPmfWtythW3r8pYKUG+hsiuQ@mail.gmail.com
Backpatch-through: 15
src/backend/backup/basebackup.c | 51 +++++++++++++++++++++++++++++------------
1 file changed, 36 insertions(+), 15 deletions(-)
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index a28b8849c19..f9400819d2f 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -81,7 +81,8 @@ static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeo
List *tablespaces, bool sendtblspclinks,
backup_manifest_info *manifest, const char *spcoid);
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
- struct stat *statbuf, bool missing_ok, Oid dboid,
+ const struct stat *statbuf, bool missing_ok, Oid dboid,
+ bool isRelationFile,
backup_manifest_info *manifest, const char *spcoid);
static void sendFileWithContent(bbsink *sink, const char *filename,
const char *content,
@@ -346,7 +347,7 @@ perform_base_backup(basebackup_options *opt, bbsink *sink)
errmsg("could not stat file \"%s\": %m",
XLOG_CONTROL_FILE)));
sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
- false, InvalidOid, &manifest, NULL);
+ false, InvalidOid, false, &manifest, NULL);
}
else
{
@@ -615,7 +616,7 @@ perform_base_backup(basebackup_options *opt, bbsink *sink)
errmsg("could not stat file \"%s\": %m", pathbuf)));
sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
- &manifest, NULL);
+ false, &manifest, NULL);
/* unconditionally mark file as archived */
StatusFilePath(pathbuf, fname, ".done");
@@ -1188,6 +1189,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
ForkNumber relForkNum; /* Type of fork if file is a relation */
int relnumchars; /* Chars in filename that are the
* relnumber */
+ bool isRelationFile = false;
/* Skip special stuff */
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
@@ -1239,10 +1241,13 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
if (excludeFound)
continue;
+ if (isDbDir)
+ isRelationFile =
+ parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
+ &relForkNum);
+
/* Exclude all forks for unlogged tables except the init fork */
- if (isDbDir &&
- parse_filename_for_nontemp_relation(de->d_name, &relnumchars,
- &relForkNum))
+ if (isRelationFile)
{
/* Never exclude init forks */
if (relForkNum != INIT_FORKNUM)
@@ -1412,7 +1417,7 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
if (!sizeonly)
sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
- manifest, spcoid);
+ isRelationFile, manifest, spcoid);
if (sent || sizeonly)
{
@@ -1476,12 +1481,16 @@ is_checksummed_file(const char *fullpath, const char *filename)
* If dboid is anything other than InvalidOid then any checksum failures
* detected will get reported to the cumulative stats system.
*
+ * For non-relation files, the stat data of the opened file is used, not
+ * 'statbuf'.
+ *
* Returns true if the file was successfully sent, false if 'missing_ok',
* and the file did not exist.
*/
static bool
sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
- struct stat *statbuf, bool missing_ok, Oid dboid,
+ const struct stat *statbuf, bool missing_ok, Oid dboid,
+ bool isRelationFile,
backup_manifest_info *manifest, const char *spcoid)
{
int fd;
@@ -1498,6 +1507,7 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
char *segmentpath;
bool verify_checksum = false;
pg_checksum_context checksum_ctx;
+ struct stat filestat;
if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
elog(ERROR, "could not initialize checksum of file \"%s\"",
@@ -1513,7 +1523,18 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
errmsg("could not open file \"%s\": %m", readfilename)));
}
- _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
+ /*
+ * Non-relation files may be replaced by a rename(), making irrelevant the
+ * caller's stat data. Reuse fstat() on the opened file.
+ */
+ filestat = *statbuf;
+
+ if (!isRelationFile && fstat(fd, &filestat) != 0)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", readfilename)));
+
+ _tarWriteHeader(sink, tarfilename, NULL, &filestat, false);
if (!noverify_checksums && DataChecksumsEnabled())
{
@@ -1552,9 +1573,9 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
* for a base backup we can ignore such extended data. It will be restored
* from WAL.
*/
- while (len < statbuf->st_size)
+ while (len < filestat.st_size)
{
- size_t remaining = statbuf->st_size - len;
+ size_t remaining = filestat.st_size - len;
/* Try to read some more data. */
cnt = basebackup_read_file(fd, sink->bbs_buffer,
@@ -1693,9 +1714,9 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
}
/* If the file was truncated while we were sending it, pad it with zeros */
- while (len < statbuf->st_size)
+ while (len < filestat.st_size)
{
- size_t remaining = statbuf->st_size - len;
+ size_t remaining = filestat.st_size - len;
size_t nbytes = Min(sink->bbs_buffer_length, remaining);
MemSet(sink->bbs_buffer, 0, nbytes);
@@ -1729,8 +1750,8 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
total_checksum_failures += checksum_failures;
- AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
- (pg_time_t) statbuf->st_mtime, &checksum_ctx);
+ AddFileToBackupManifest(manifest, spcoid, tarfilename, filestat.st_size,
+ (pg_time_t) filestat.st_mtime, &checksum_ctx);
return true;
}
[parent: 658e15092503]