pg.ddx.io pgbackrest/pgbackrest GitHub issues and pull requests
help / color / mirror / Atom feed[pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
10+ messages / 2 participants
[nested] [flat]
* [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-04 15:32 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
0 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-04 15:32 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
A reconnect frees the libssh2 session while other handles may still be open on it and the next use of one of those handles reads freed memory. For example, a block incremental restore holds the next file's handle open in the multi-read while super blocks are read so a lost connection detected when opening a super block read leaves that handle invalid.
Add a connection id to the SFTP storage that is incremented when the session is freed and store it when a file is opened. A read or write on a handle from a lost connection now throws an error, which allows the job to be retried. Closing such a read is skipped. Closing such a write throws an error because the written data may not have reached the server.
Writes need the check because the remote process does all reads and writes through one storage object so a read open can reconnect while a bundle write is open.
inline patch 1: src/storage/sftp/read.c, src/storage/sftp/storage.c, src/storage/sftp/storage.intern.h and 2 more (+63/-1 lines) download inline-1.patch
diff --git a/src/storage/sftp/read.c b/src/storage/sftp/read.c
index d84b2e6c48..2596679e48 100644
--- a/src/storage/sftp/read.c
+++ b/src/storage/sftp/read.c
@@ -24,6 +24,7 @@ struct StorageReadSftp
const Variant *limit; // Read limit (NULL for no limit)
LIBSSH2_SFTP_HANDLE *sftpHandle; // LibSsh2 session sftp handle
+ unsigned int connId; // Id of the connection the handle was opened on
LIBSSH2_SFTP_ATTRIBUTES *attr; // LibSsh2 file attributes
uint64_t current; // Current bytes read from file
bool eof; // Did we reach end of file
@@ -75,6 +76,8 @@ storageReadSftpOpen(THIS_VOID)
// Else success
else
{
+ this->connId = storageSftpConnId(this->storage);
+
// Seek to offset, libssh2_sftp_seek64 returns void
if (this->offset != 0)
libssh2_sftp_seek64(this->sftpHandle, this->offset);
@@ -105,6 +108,10 @@ storageReadSftp(THIS_VOID, Buffer *const buffer, const bool block)
// Read if EOF has not been reached
if (!this->eof)
{
+ // Error when the connection the handle was opened on has been lost
+ if (this->connId != storageSftpConnId(this->storage))
+ THROW_FMT(FileReadError, "unable to read '%s': connection was lost", strZ(this->name));
+
// Rather than enable/disable limit checking just use a big number when there is no limit. We can feel pretty confident that
// no files will be > UINT64_MAX in size.
const uint64_t limit = this->limit == NULL ? UINT64_MAX : varUInt64(this->limit);
@@ -191,7 +198,8 @@ storageReadSftpClose(THIS_VOID)
ASSERT(this != NULL);
- if (this->sftpHandle != NULL)
+ // Skip the close when the connection the handle was opened on has been lost
+ if (this->sftpHandle != NULL && this->connId == storageSftpConnId(this->storage))
{
int rc;
diff --git a/src/storage/sftp/storage.c b/src/storage/sftp/storage.c
index f616a31e3e..e672cba34d 100644
--- a/src/storage/sftp/storage.c
+++ b/src/storage/sftp/storage.c
@@ -37,6 +37,7 @@ struct StorageSftp
IoSession *ioSession; // IoSession (socket) connection to SFTP server
LIBSSH2_SESSION *session; // LibSsh2 session
LIBSSH2_SFTP *sftpSession; // LibSsh2 session sftp session
+ unsigned int connId; // Connection id, incremented when the session is freed
TimeMSec timeout; // Session timeout
// Connection parameters retained so the session can be reopened after the server drops an idle connection
@@ -1189,6 +1190,18 @@ storageSftpSessionSftp(StorageSftp *const this)
FUNCTION_TEST_RETURN_TYPE_P(LIBSSH2_SFTP, this->sftpSession);
}
+FN_EXTERN unsigned int
+storageSftpConnId(StorageSftp *const this)
+{
+ FUNCTION_TEST_BEGIN();
+ FUNCTION_TEST_PARAM(STORAGE_SFTP, this);
+ FUNCTION_TEST_END();
+
+ ASSERT(this != NULL);
+
+ FUNCTION_TEST_RETURN(UINT, this->connId);
+}
+
/***********************************************************************************************************************************
Is the libssh2 error a sign that the connection to the server has been lost? Socket-level errors always mean the connection is gone.
LIBSSH2_ERROR_SOCKET_SEND is intentionally excluded since a failed send is ambiguous (and is used throughout the unit tests as a
@@ -1261,6 +1274,7 @@ storageSftpConnFree(StorageSftp *const this)
this->session = NULL;
this->sftpSession = NULL;
+ this->connId++;
// Free the socket session, which closes the file descriptor
ioSessionFree(this->ioSession);
diff --git a/src/storage/sftp/storage.intern.h b/src/storage/sftp/storage.intern.h
index 1a1af077e1..e7250a6895 100644
--- a/src/storage/sftp/storage.intern.h
+++ b/src/storage/sftp/storage.intern.h
@@ -21,6 +21,9 @@ Getters
FN_EXTERN LIBSSH2_SESSION *storageSftpSession(StorageSftp *this);
FN_EXTERN LIBSSH2_SFTP *storageSftpSessionSftp(StorageSftp *this);
+// Id of the current connection
+FN_EXTERN unsigned int storageSftpConnId(StorageSftp *this);
+
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
diff --git a/src/storage/sftp/write.c b/src/storage/sftp/write.c
index 0e65a0987a..559ea03e0f 100644
--- a/src/storage/sftp/write.c
+++ b/src/storage/sftp/write.c
@@ -27,6 +27,7 @@ struct StorageWriteSftp
const String *nameTmp; // Temporary filename utilized for atomic ops
const String *path; // Utilized for path operations
LIBSSH2_SFTP_HANDLE *sftpHandle; // LibSsh2 session sftp handle
+ unsigned int connId; // Id of the connection the handle was opened on
};
/***********************************************************************************************************************************
@@ -100,6 +101,8 @@ storageWriteSftpOpen(THIS_VOID)
storageSftpEvalLibSsh2Error(rc, sftpErr, &FileOpenError, strNewFmt(STORAGE_ERROR_WRITE_OPEN, strZ(this->name)), NULL);
}
+ this->connId = storageSftpConnId(this->storage);
+
FUNCTION_LOG_RETURN_VOID();
}
@@ -120,6 +123,10 @@ storageWriteSftp(THIS_VOID, const Buffer *const buffer)
ASSERT(buffer != NULL);
ASSERT(this->sftpHandle != NULL);
+ // Error when the connection the handle was opened on has been lost
+ if (this->connId != storageSftpConnId(this->storage))
+ THROW_FMT(FileWriteError, "unable to write '%s': connection was lost", strZ(this->nameTmp));
+
ssize_t rc;
size_t remains = bufUsed(buffer); // Amount left to write
size_t offset = 0; // Offset into the buffer
@@ -249,6 +256,10 @@ storageWriteSftpClose(THIS_VOID)
// Close if the file has not already been closed
if (this->sftpHandle != NULL)
{
+ // Error when the connection the handle was opened on has been lost
+ if (this->connId != storageSftpConnId(this->storage))
+ THROW_FMT(FileCloseError, STORAGE_ERROR_WRITE_CLOSE ": connection was lost", strZ(this->nameTmp));
+
int rc;
if (this->syncFile)
diff --git a/test/src/module/storage/sftpTest.c b/test/src/module/storage/sftpTest.c
index ecc370fa23..c52d06aa16 100644
--- a/test/src/module/storage/sftpTest.c
+++ b/test/src/module/storage/sftpTest.c
@@ -2693,6 +2693,8 @@ testRun(void)
TEST_TITLE("read open reconnects after the connection is lost");
HRN_LIBSSH2_SCRIPT_SET(
+ // File left open across the reconnect
+ HRN_LIBSSH2_OPEN_READ(TEST_PATH "/readtest.txt"),
// First open fails because the server dropped the idle connection
HRN_LIBSSH2_OPEN_READ(TEST_PATH "/readtest.txt", .resultNull = true),
HRN_LIBSSH2_ERRNO(LIBSSH2_ERROR_BAD_USE),
@@ -2712,10 +2714,20 @@ testRun(void)
HRN_LIBSSH2_OPEN_READ(TEST_PATH "/readtest.txt"),
HRN_LIBSSH2_CLOSE());
+ StorageRead *fileStale = NULL;
+
+ TEST_ASSIGN(fileStale, storageNewReadP(storageTest, fileName), "new read file left open");
+ TEST_RESULT_BOOL(ioReadOpen(storageReadIo(fileStale)), true, "open file before reconnect");
+
TEST_ASSIGN(file, storageNewReadP(storageTest, fileName), "new read file");
TEST_RESULT_BOOL(ioReadOpen(storageReadIo(file)), true, "open file after reconnect");
TEST_RESULT_VOID(ioReadClose(storageReadIo(file)), "close file");
+ TEST_ERROR(
+ ioRead(storageReadIo(fileStale), outBuffer), FileReadError,
+ "unable to read '" TEST_PATH "/readtest.txt': connection was lost");
+ TEST_RESULT_VOID(ioReadClose(storageReadIo(fileStale)), "close skipped when connection was lost");
+
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("read open errors when the open fails again after reconnect (only one retry)");
@@ -2896,6 +2908,8 @@ testRun(void)
TEST_TITLE("write open reconnects after the connection is lost");
HRN_LIBSSH2_SCRIPT_SET(
+ // File left open across the reconnect
+ HRN_LIBSSH2_OPEN_WRITE(TEST_PATH "/sub1/testfile.pgbackrest.tmp"),
// First open fails because the server dropped the idle connection
HRN_LIBSSH2_OPEN_WRITE(TEST_PATH "/sub1/testfile.pgbackrest.tmp", .resultNull = true),
HRN_LIBSSH2_ERRNO(LIBSSH2_ERROR_SOCKET_DISCONNECT),
@@ -2907,10 +2921,22 @@ testRun(void)
HRN_LIBSSH2_CLOSE(),
HRN_LIBSSH2_RENAME(TEST_PATH "/sub1/testfile.pgbackrest.tmp", TEST_PATH "/sub1/testfile"));
+ StorageWrite *fileStale = NULL;
+
+ TEST_ASSIGN(fileStale, storageNewWriteP(storageTest, fileName), "new write file left open");
+ TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(fileStale)), "open file before reconnect");
+
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileName), "new write file");
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(file)), "open file after reconnect");
TEST_RESULT_VOID(ioWriteClose(storageWriteIo(file)), "close file");
+ TEST_ERROR(
+ storageWriteSftp(fileStale->driver, BUFSTRDEF("X")), FileWriteError,
+ "unable to write '" TEST_PATH "/sub1/testfile.pgbackrest.tmp': connection was lost");
+ TEST_ERROR(
+ storageWriteSftpClose(fileStale->driver), FileCloseError,
+ "unable to close file '" TEST_PATH "/sub1/testfile.pgbackrest.tmp' after write: connection was lost");
+
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write open errors when the open fails again after reconnect (only one retry)");
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-04 15:32 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-04 15:32 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele assigned @dwsteele.
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-04 15:32 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-04 15:32 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele applied label "bug".
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-04 15:32 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-04 15:32 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele applied label "module [core]".
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-04 15:32 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-04 15:32 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele set milestone to "2.61.0".
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903 (approved): Prevent use of SFTP handles after a reconnect.
@ 2026-11-05 14:03 "hunleyd (@hunleyd)" <noreply+hunleyd@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: hunleyd (@hunleyd) @ 2026-11-05 14:03 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-06 08:59 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-06 08:59 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele merged this (commit 82c0182fb56f).
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-06 08:59 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-06 08:59 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele closed this.
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-10 12:46 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-10 12:46 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele removed milestone.
^ permalink raw [nested|flat] 10+ messages in thread
* Re: [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect.
@ 2026-11-10 12:46 "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
8 siblings, 0 replies; 10+ messages in thread
From: dwsteele (@dwsteele) @ 2026-11-10 12:46 UTC (permalink / raw)
To: pgbackrest/pgbackrest <noreply+pgbackrest-pgbackrest@github.com>
@dwsteele set milestone to "2.60.1".
^ permalink raw [nested|flat] 10+ messages in thread
end of thread, other threads:[~2026-11-10 12:46 UTC | newest]
Thread overview: 10+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2026-11-04 15:32 [pgbackrest/pgbackrest] PR #2903: Prevent use of SFTP handles after a reconnect. "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-04 15:32 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-04 15:32 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-04 15:32 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-04 15:32 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-05 14:03 ` Re: [pgbackrest/pgbackrest] PR #2903 (approved): Prevent use of SFTP handles after a reconnect. "hunleyd (@hunleyd)" <noreply+hunleyd@github.com>
2026-11-06 08:59 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-06 08:59 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-10 12:46 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
2026-11-10 12:46 ` "dwsteele (@dwsteele)" <noreply+dwsteele@github.com>
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox