agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v9 4/4] Change policy of XLog read-buffer allocation
14+ messages / 2 participants
[nested] [flat]

* [PATCH v11 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b6c7125aa1..e69b6abae7 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9acc8d03e2..cb47aa239a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c7c05edcda..5a40047733 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1035,6 +1035,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1115,6 +1116,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Wed_Nov_27_12_09_23_2019_240)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v10 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 8630ad2032..7f394156bd 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 694307a177..b144e3e241 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6477,6 +6477,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7889,6 +7890,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index bdde75086c..71a4931131 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -110,7 +110,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -123,7 +122,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -150,7 +148,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d99474c173..18d3248192 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cad8e13965..c6247344d6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_May_26_16_40_02_2020_373)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v9 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e8a4c7916b..0bc7549b07 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6355,6 +6355,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7739,6 +7740,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ce7fd55496..12a097c3eb 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -104,7 +104,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -117,7 +116,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -141,7 +139,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Thu_Oct_24_14_51_01_2019_720)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v5 3/3] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     |  2 ++
 src/backend/access/transam/xlog.c         |  2 ++
 src/backend/access/transam/xlogreader.c   | 18 ------------------
 src/backend/replication/logical/logical.c |  2 ++
 src/bin/pg_rewind/parsexlog.c             |  6 ++++++
 src/bin/pg_waldump/pg_waldump.c           |  2 ++
 6 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 929da4eef2..9ec88b35ef 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 96e2115aad..425f7a12bd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6351,6 +6351,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7722,6 +7723,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 66bd9eb8d7..26f6834b8f 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -79,27 +79,11 @@ XLogReaderAllocate(int wal_segment_size)
 
 	state->max_block_id = -1;
 
-	/*
-	 * Permanently allocate readBuf.  We do it this way, rather than just
-	 * making a static array, for two reasons: (1) no need to waste the
-	 * storage in most instantiations of the backend; (2) a static char array
-	 * isn't guaranteed to have any particular alignment, whereas
-	 * palloc_extended() will provide MAXALIGN'd storage.
-	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
-											  MCXT_ALLOC_NO_OOM);
-	if (!state->readBuf)
-	{
-		pfree(state);
-		return NULL;
-	}
-
 	state->wal_segment_size = wal_segment_size;
 	state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -112,7 +96,6 @@ XLogReaderAllocate(int wal_segment_size)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -136,7 +119,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 56e2f8b0b0..bdf3c81b03 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1190,6 +1191,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Fri_Sep_06_16_33_18_2019_198)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v8 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9cab82e76d..80434aed2e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Fri_Sep_27_12_07_26_2019_308)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v13 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ce62559f42..e319793404 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6366,6 +6366,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7753,6 +7754,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d0a002fae7..9f67674a88 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -516,6 +517,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index e82f184de2..a40f1d71e4 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1032,6 +1032,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1108,6 +1109,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Jan_28_21_20_20_2020_871)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v12 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f519a5b6a2..e72d5c7ccb 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 71984f00f7..81a95623a2 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 16ee68efbe..4a6491007f 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1036,6 +1036,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1116,6 +1117,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Fri_Nov_29_17_14_21_2019_330)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v8 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 5b16681a37..c8b6c5ee90 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -132,6 +135,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -173,6 +177,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -223,6 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Apr_21_17_04_27_2020_275)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v9 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 86b6540057..78073f969d 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Wed_Apr_22_10_12_46_2020_740)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v7 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a3eded30ad..3f7f3e4d4f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6411,6 +6411,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7813,6 +7814,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0cd0d132af..ac6740f21d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1033,6 +1033,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1109,6 +1110,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Mar_24_18_24_13_2020_275)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v6 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     |  2 ++
 src/backend/access/transam/xlog.c         |  2 ++
 src/backend/access/transam/xlogreader.c   | 18 ------------------
 src/backend/replication/logical/logical.c |  2 ++
 src/bin/pg_rewind/parsexlog.c             |  6 ++++++
 src/bin/pg_waldump/pg_waldump.c           |  2 ++
 6 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index a736504d62..358951c232 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c20d3b1418..a199600399 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index e00fa270ae..1f98dd0168 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -80,27 +80,11 @@ XLogReaderAllocate(int wal_segment_size)
 
 	state->max_block_id = -1;
 
-	/*
-	 * Permanently allocate readBuf.  We do it this way, rather than just
-	 * making a static array, for two reasons: (1) no need to waste the
-	 * storage in most instantiations of the backend; (2) a static char array
-	 * isn't guaranteed to have any particular alignment, whereas
-	 * palloc_extended() will provide MAXALIGN'd storage.
-	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
-											  MCXT_ALLOC_NO_OOM);
-	if (!state->readBuf)
-	{
-		pfree(state);
-		return NULL;
-	}
-
 	state->wal_segment_size = wal_segment_size;
 	state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -113,7 +97,6 @@ XLogReaderAllocate(int wal_segment_size)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -137,7 +120,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index a61a5a91cb..5eba802720 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1188,6 +1189,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Tue_Sep_10_17_40_54_2019_177)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v15 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b0d60a0d0f..5955bab699 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 06e7ff4a24..04478e78bc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6480,6 +6480,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7892,6 +7893,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9281b57379..388bddaf41 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -112,7 +112,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -125,7 +124,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -152,7 +150,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index a15b0b3355..31f1029000 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 3dd6df4be6..26cba673f6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.4


----Next_Part(Thu_Jul__2_13_53_30_2020_072)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v7 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29 Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9932ba3882..b8ad4ffcd7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Wed_Sep_25_15_50_32_2019_576)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v50 8/8] Reserve replication slots specifically for REPACK
@ 2026-04-01 17:54 Álvaro Herrera <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Álvaro Herrera @ 2026-04-01 17:54 UTC (permalink / raw)

This allows REPACK to not interfere with other operations that use
replication slots.  This eases configurability.
---
 doc/src/sgml/config.sgml                      | 16 ++++
 doc/src/sgml/ref/repack.sgml                  |  6 +-
 src/backend/commands/repack.c                 |  3 +-
 src/backend/commands/repack_worker.c          |  4 +-
 src/backend/replication/logical/launcher.c    |  2 +-
 src/backend/replication/logical/slotsync.c    |  5 +-
 src/backend/replication/slot.c                | 78 ++++++++++++-------
 src/backend/replication/slotfuncs.c           |  8 +-
 src/backend/replication/walsender.c           |  4 +-
 src/backend/utils/misc/guc_parameters.dat     |  8 ++
 src/backend/utils/misc/postgresql.conf.sample |  2 +
 src/include/replication/slot.h                |  6 +-
 12 files changed, 96 insertions(+), 46 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index fdb77df0fdb..a87626a01b6 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4638,6 +4638,22 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"'  # Windows
        </listitem>
       </varlistentry>
 
+      <varlistentry id="guc-max-repack-replication-slots" xreflabel="max_repack_replication_slots">
+       <term><varname>max_repack_replication_slots</varname> (<type>integer</type>)
+       <indexterm>
+        <primary><varname>max_repack_replication_slots</varname> configuration parameter</primary>
+       </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies the maximum number of replication slots for use of
+         the <command>REPACK</command> command.  The default is 5.
+         This parameter can only be set at server start.
+        </para>
+       </listitem>
+      </varlistentry>
+
+
       <varlistentry id="guc-max-replication-slots" xreflabel="max_replication_slots">
        <term><varname>max_replication_slots</varname> (<type>integer</type>)
        <indexterm>
diff --git a/doc/src/sgml/ref/repack.sgml b/doc/src/sgml/ref/repack.sgml
index bec18c44cc8..1424446ba7c 100644
--- a/doc/src/sgml/ref/repack.sgml
+++ b/doc/src/sgml/ref/repack.sgml
@@ -293,9 +293,9 @@ REPACK [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] USING
 
        <listitem>
         <para>
-         The <link linkend="guc-max-replication-slots"><varname>max_replication_slots</varname></link>
-         configuration parameter does not allow for creation of an additional
-         replication slot.
+         The <link linkend="guc-max-repack-replication-slots"><varname>max_repack_replication_slots</varname></link>
+         configuration parameter does not allow for the creation of an
+         additional replication slot.
         </para>
        </listitem>
       </itemizedlist>
diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c
index d6e446d582d..2fc30008f59 100644
--- a/src/backend/commands/repack.c
+++ b/src/backend/commands/repack.c
@@ -3383,7 +3383,8 @@ start_repack_decoding_worker(Oid relid)
 		ereport(ERROR,
 				errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
 				errmsg("out of background worker slots"),
-				errhint("You might need to increase \"%s\".", "max_worker_processes"));
+		/* FIXME rename to max_repack_processes? */
+				errhint("You might need to increase \"%s\".", "max_repack_replication_slots"));
 
 	decoding_worker->seg = seg;
 	decoding_worker->error_mqh = mqh;
diff --git a/src/backend/commands/repack_worker.c b/src/backend/commands/repack_worker.c
index ff34e246469..610592a05b0 100644
--- a/src/backend/commands/repack_worker.c
+++ b/src/backend/commands/repack_worker.c
@@ -233,8 +233,8 @@ repack_setup_logical_decoding(Oid relid)
 	 * RS_TEMPORARY so that the slot gets cleaned up on ERROR.
 	 */
 	snprintf(NameStr(slotname), NAMEDATALEN, "repack_%d", MyProcPid);
-	ReplicationSlotCreate(NameStr(slotname), true, RS_TEMPORARY, false, false,
-						  false);
+	ReplicationSlotCreate(NameStr(slotname), true, RS_TEMPORARY, false, true,
+						  false, false);
 
 	EnsureLogicalDecodingEnabled();
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 09964198550..d83125afd0d 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -1575,7 +1575,7 @@ CreateConflictDetectionSlot(void)
 			errmsg("creating replication conflict detection slot"));
 
 	ReplicationSlotCreate(CONFLICT_DETECTION_SLOT, false, RS_PERSISTENT, false,
-						  false, false);
+						  false, false, false);
 
 	init_conflict_slot_xmin();
 }
diff --git a/src/backend/replication/logical/slotsync.c b/src/backend/replication/logical/slotsync.c
index e75db69e3f6..1bc7f3f600d 100644
--- a/src/backend/replication/logical/slotsync.c
+++ b/src/backend/replication/logical/slotsync.c
@@ -425,7 +425,7 @@ get_local_synced_slots(void)
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 
-	for (int i = 0; i < max_replication_slots; i++)
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
@@ -814,6 +814,7 @@ synchronize_one_slot(RemoteSlot *remote_slot, Oid remote_dbid,
 		 */
 		ReplicationSlotCreate(remote_slot->name, true, RS_TEMPORARY,
 							  remote_slot->two_phase,
+							  false,
 							  remote_slot->failover,
 							  true);
 
@@ -1691,7 +1692,7 @@ update_synced_slots_inactive_since(void)
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 
-	for (int i = 0; i < max_replication_slots; i++)
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
diff --git a/src/backend/replication/slot.c b/src/backend/replication/slot.c
index d553bd5dbff..49fb10df038 100644
--- a/src/backend/replication/slot.c
+++ b/src/backend/replication/slot.c
@@ -152,6 +152,9 @@ ReplicationSlot *MyReplicationSlot = NULL;
 /* GUC variables */
 int			max_replication_slots = 10; /* the maximum number of replication
 										 * slots */
+int			max_repack_replication_slots = 5;	/* the maximum number of slots
+												 * for REPACK */
+int			TotalMaxReplicationSlots = 0;	/* sum of both */
 
 /*
  * Invalidate replication slots that have remained idle longer than this
@@ -191,12 +194,14 @@ ReplicationSlotsShmemSize(void)
 {
 	Size		size = 0;
 
-	if (max_replication_slots == 0)
+	TotalMaxReplicationSlots = max_replication_slots + max_repack_replication_slots;
+
+	if (TotalMaxReplicationSlots == 0)
 		return size;
 
 	size = offsetof(ReplicationSlotCtlData, replication_slots);
 	size = add_size(size,
-					mul_size(max_replication_slots, sizeof(ReplicationSlot)));
+					mul_size(TotalMaxReplicationSlots, sizeof(ReplicationSlot)));
 
 	return size;
 }
@@ -209,7 +214,7 @@ ReplicationSlotsShmemInit(void)
 {
 	bool		found;
 
-	if (max_replication_slots == 0)
+	if (TotalMaxReplicationSlots == 0)
 		return;
 
 	ReplicationSlotCtl = (ReplicationSlotCtlData *)
@@ -223,7 +228,7 @@ ReplicationSlotsShmemInit(void)
 		/* First time through, so initialize */
 		MemSet(ReplicationSlotCtl, 0, ReplicationSlotsShmemSize());
 
-		for (i = 0; i < max_replication_slots; i++)
+		for (i = 0; i < TotalMaxReplicationSlots; i++)
 		{
 			ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[i];
 
@@ -373,6 +378,7 @@ IsSlotForConflictCheck(const char *name)
  * db_specific: logical decoding is db specific; if the slot is going to
  *	   be used for that pass true, otherwise false.
  * two_phase: If enabled, allows decoding of prepared transactions.
+ * repack: If true, use a slot from the pool for REPACK.
  * failover: If enabled, allows the slot to be synced to standbys so
  *     that logical replication can be resumed after failover.
  * synced: True if the slot is synchronized from the primary server.
@@ -380,10 +386,11 @@ IsSlotForConflictCheck(const char *name)
 void
 ReplicationSlotCreate(const char *name, bool db_specific,
 					  ReplicationSlotPersistency persistency,
-					  bool two_phase, bool failover, bool synced)
+					  bool two_phase, bool repack, bool failover, bool synced)
 {
 	ReplicationSlot *slot = NULL;
-	int			i;
+	int			startpoint,
+				endpoint;
 
 	Assert(MyReplicationSlot == NULL);
 
@@ -432,12 +439,16 @@ ReplicationSlotCreate(const char *name, bool db_specific,
 	LWLockAcquire(ReplicationSlotAllocationLock, LW_EXCLUSIVE);
 
 	/*
-	 * Check for name collision, and identify an allocatable slot.  We need to
-	 * hold ReplicationSlotControlLock in shared mode for this, so that nobody
-	 * else can change the in_use flags while we're looking at them.
+	 * Check for name collision (across the whole array), and identify an
+	 * allocatable slot (in the array slice specific to our current use case:
+	 * either general, or REPACK only).  We need to hold
+	 * ReplicationSlotControlLock in shared mode for this, so that nobody else
+	 * can change the in_use flags while we're looking at them.
 	 */
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (i = 0; i < max_replication_slots; i++)
+	startpoint = repack ? max_replication_slots : 0;
+	endpoint = repack ? TotalMaxReplicationSlots : max_replication_slots;
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
@@ -445,7 +456,9 @@ ReplicationSlotCreate(const char *name, bool db_specific,
 			ereport(ERROR,
 					(errcode(ERRCODE_DUPLICATE_OBJECT),
 					 errmsg("replication slot \"%s\" already exists", name)));
-		if (!s->in_use && slot == NULL)
+
+		if (i >= startpoint && i < endpoint &&
+			!s->in_use && slot == NULL)
 			slot = s;
 	}
 	LWLockRelease(ReplicationSlotControlLock);
@@ -548,7 +561,7 @@ SearchNamedReplicationSlot(const char *name, bool need_lock)
 	if (need_lock)
 		LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
@@ -576,7 +589,7 @@ int
 ReplicationSlotIndex(ReplicationSlot *slot)
 {
 	Assert(slot >= ReplicationSlotCtl->replication_slots &&
-		   slot < ReplicationSlotCtl->replication_slots + max_replication_slots);
+		   slot < ReplicationSlotCtl->replication_slots + TotalMaxReplicationSlots);
 
 	return slot - ReplicationSlotCtl->replication_slots;
 }
@@ -870,7 +883,7 @@ ReplicationSlotCleanup(bool synced_only)
 restart:
 	found_valid_logicalslot = false;
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
@@ -1252,7 +1265,7 @@ ReplicationSlotsComputeRequiredXmin(bool already_locked)
 	if (!already_locked)
 		LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 		TransactionId effective_xmin;
@@ -1307,7 +1320,7 @@ ReplicationSlotsComputeRequiredLSN(void)
 	Assert(ReplicationSlotCtl != NULL);
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 		XLogRecPtr	restart_lsn;
@@ -1374,12 +1387,12 @@ ReplicationSlotsComputeLogicalRestartLSN(void)
 	XLogRecPtr	result = InvalidXLogRecPtr;
 	int			i;
 
-	if (max_replication_slots <= 0)
+	if (TotalMaxReplicationSlots <= 0)
 		return InvalidXLogRecPtr;
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
 
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s;
 		XLogRecPtr	restart_lsn;
@@ -1454,11 +1467,11 @@ ReplicationSlotsCountDBSlots(Oid dboid, int *nslots, int *nactive)
 
 	*nslots = *nactive = 0;
 
-	if (max_replication_slots <= 0)
+	if (TotalMaxReplicationSlots <= 0)
 		return false;
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s;
 
@@ -1515,13 +1528,13 @@ ReplicationSlotsDropDBSlots(Oid dboid)
 	bool		found_valid_logicalslot;
 	bool		dropped = false;
 
-	if (max_replication_slots <= 0)
+	if (TotalMaxReplicationSlots <= 0)
 		return;
 
 restart:
 	found_valid_logicalslot = false;
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s;
 		char	   *slotname;
@@ -1618,11 +1631,11 @@ CheckLogicalSlotExists(void)
 {
 	bool		found = false;
 
-	if (max_replication_slots <= 0)
+	if (TotalMaxReplicationSlots <= 0)
 		return false;
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (int i = 0; i < max_replication_slots; i++)
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s;
 		bool		invalidated;
@@ -1663,7 +1676,11 @@ CheckSlotRequirements(void)
 	 * needs the same check.
 	 */
 
-	if (max_replication_slots == 0)
+	/*
+	 * FIXME how to relax this for repack in a way that doesn't mess
+	 * everything up?
+	 */
+	if (TotalMaxReplicationSlots == 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 				 errmsg("replication slots can only be used if \"max_replication_slots\" > 0")));
@@ -2224,7 +2241,7 @@ InvalidateObsoleteReplicationSlots(uint32 possible_causes,
 	Assert(!(possible_causes & RS_INVAL_WAL_REMOVED) || oldestSegno > 0);
 	Assert(possible_causes != RS_INVAL_NONE);
 
-	if (max_replication_slots == 0)
+	if (TotalMaxReplicationSlots == 0)
 		return invalidated;
 
 	XLogSegNoOffsetToRecPtr(oldestSegno, 0, wal_segment_size, oldestLSN);
@@ -2232,7 +2249,7 @@ InvalidateObsoleteReplicationSlots(uint32 possible_causes,
 restart:
 	found_valid_logicalslot = false;
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (int i = 0; i < max_replication_slots; i++)
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 		bool		released_lock = false;
@@ -2337,7 +2354,7 @@ CheckPointReplicationSlots(bool is_shutdown)
 	 */
 	LWLockAcquire(ReplicationSlotAllocationLock, LW_SHARED);
 
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 		char		path[MAXPGPATH];
@@ -2438,7 +2455,7 @@ StartupReplicationSlots(void)
 	FreeDir(replication_dir);
 
 	/* currently no slots exist, we're done. */
-	if (max_replication_slots <= 0)
+	if (TotalMaxReplicationSlots <= 0)
 		return;
 
 	/* Now that we have recovered all the data, compute replication xmin */
@@ -2868,7 +2885,7 @@ RestoreSlotFromDisk(const char *name)
 				 errhint("Change \"wal_level\" to be \"replica\" or higher.")));
 
 	/* nothing can be active yet, don't lock anything */
-	for (i = 0; i < max_replication_slots; i++)
+	for (i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *slot;
 
@@ -2910,6 +2927,7 @@ RestoreSlotFromDisk(const char *name)
 		break;
 	}
 
+	/* XXX might be misleading if the slots previously in use were REPACK. */
 	if (!restored)
 		ereport(FATAL,
 				(errmsg("too many replication slots active before shutdown"),
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 9f5e4f998fe..cdb4dbd87c9 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -53,7 +53,7 @@ create_physical_replication_slot(char *name, bool immediately_reserve,
 	/* acquire replication slot, this will check for conflicting names */
 	ReplicationSlotCreate(name, false,
 						  temporary ? RS_TEMPORARY : RS_PERSISTENT, false,
-						  false, false);
+						  false, false, false);
 
 	if (immediately_reserve)
 	{
@@ -146,7 +146,7 @@ create_logical_replication_slot(char *name, char *plugin,
 	 */
 	ReplicationSlotCreate(name, true,
 						  temporary ? RS_TEMPORARY : RS_EPHEMERAL, two_phase,
-						  failover, false);
+						  false, failover, false);
 
 	/*
 	 * Ensure the logical decoding is enabled before initializing the logical
@@ -270,7 +270,7 @@ pg_get_replication_slots(PG_FUNCTION_ARGS)
 	currlsn = GetXLogWriteRecPtr();
 
 	LWLockAcquire(ReplicationSlotControlLock, LW_SHARED);
-	for (slotno = 0; slotno < max_replication_slots; slotno++)
+	for (slotno = 0; slotno < TotalMaxReplicationSlots; slotno++)
 	{
 		ReplicationSlot *slot = &ReplicationSlotCtl->replication_slots[slotno];
 		ReplicationSlot slot_contents;
@@ -665,7 +665,7 @@ copy_replication_slot(FunctionCallInfo fcinfo, bool logical_slot)
 	 * managed to create the new slot, we advance the new slot's restart_lsn
 	 * to the source slot's updated restart_lsn the second time we lock it.
 	 */
-	for (int i = 0; i < max_replication_slots; i++)
+	for (int i = 0; i < TotalMaxReplicationSlots; i++)
 	{
 		ReplicationSlot *s = &ReplicationSlotCtl->replication_slots[i];
 
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 2bb3f34dc6d..75ef3419a15 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1220,7 +1220,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
 	{
 		ReplicationSlotCreate(cmd->slotname, false,
 							  cmd->temporary ? RS_TEMPORARY : RS_PERSISTENT,
-							  false, false, false);
+							  false, false, false, false);
 
 		if (reserve_wal)
 		{
@@ -1251,7 +1251,7 @@ CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
 		 */
 		ReplicationSlotCreate(cmd->slotname, true,
 							  cmd->temporary ? RS_TEMPORARY : RS_EPHEMERAL,
-							  two_phase, failover, false);
+							  two_phase, false, failover, false);
 
 		/*
 		 * Do options check early so that we can bail before calling the
diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat
index e556b8844d8..8eb251659b0 100644
--- a/src/backend/utils/misc/guc_parameters.dat
+++ b/src/backend/utils/misc/guc_parameters.dat
@@ -2070,6 +2070,14 @@
   max => 'MAX_BACKENDS',
 },
 
+{ name => 'max_repack_replication_slots', type => 'int', context => 'PGC_POSTMASTER', group => 'REPLICATION_SENDING',
+  short_desc => 'Sets the maximum number of replication slots for use by REPACK.',
+  variable => 'max_repack_replication_slots',
+  boot_val => '5',
+  min => '0',
+  max => 'MAX_BACKENDS',
+},
+
 /* see max_wal_senders */
 { name => 'max_replication_slots', type => 'int', context => 'PGC_POSTMASTER', group => 'REPLICATION_SENDING',
   short_desc => 'Sets the maximum number of simultaneously defined replication slots.',
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 2c5e98d1d4d..9e9a390a01b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -348,6 +348,8 @@
                                 # (change requires restart)
 #max_replication_slots = 10     # max number of replication slots
                                 # (change requires restart)
+#max_repack_replication_slots = 5 # max number of replication slots for REPACK
+                                # (change requires restart)
 #wal_keep_size = 0              # in megabytes; 0 disables
 #max_slot_wal_keep_size = -1    # in megabytes; -1 disables
 #idle_replication_slot_timeout = 0      # in seconds; 0 disables
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 4b4709f6e2c..88953576894 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -324,9 +324,13 @@ extern PGDLLIMPORT ReplicationSlot *MyReplicationSlot;
 
 /* GUCs */
 extern PGDLLIMPORT int max_replication_slots;
+extern PGDLLIMPORT int max_repack_replication_slots;
 extern PGDLLIMPORT char *synchronized_standby_slots;
 extern PGDLLIMPORT int idle_replication_slot_timeout_secs;
 
+/* only for slotfuncs.c, slotsync.c etc */
+extern int TotalMaxReplicationSlots;
+
 /* shmem initialization functions */
 extern Size ReplicationSlotsShmemSize(void);
 extern void ReplicationSlotsShmemInit(void);
@@ -334,7 +338,7 @@ extern void ReplicationSlotsShmemInit(void);
 /* management of individual slots */
 extern void ReplicationSlotCreate(const char *name, bool db_specific,
 								  ReplicationSlotPersistency persistency,
-								  bool two_phase, bool failover,
+								  bool two_phase, bool repack, bool failover,
 								  bool synced);
 extern void ReplicationSlotPersist(void);
 extern void ReplicationSlotDrop(const char *name, bool nowait);
-- 
2.47.3


--brnevsqjnuzpyok4--





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

end of thread, other threads:[~2026-04-01 17:54 UTC | newest]

Thread overview: 14+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-05 12:29 [PATCH v11 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v10 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v9 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v5 3/3] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v8 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v13 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v12 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v8 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v9 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v7 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v6 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v15 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v7 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2026-04-01 17:54 [PATCH v50 8/8] Reserve replication slots specifically for REPACK Álvaro Herrera <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox