agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v9 4/4] Change policy of XLog read-buffer allocation
14+ messages / 2 participants
[nested] [flat]

* [PATCH v9 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e8a4c7916b..0bc7549b07 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6355,6 +6355,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7739,6 +7740,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ce7fd55496..12a097c3eb 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -104,7 +104,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -117,7 +116,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -141,7 +139,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Thu_Oct_24_14_51_01_2019_720)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v7 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a3eded30ad..3f7f3e4d4f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6411,6 +6411,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7813,6 +7814,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 0cd0d132af..ac6740f21d 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1033,6 +1033,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1109,6 +1110,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Mar_24_18_24_13_2020_275)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v5 3/3] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     |  2 ++
 src/backend/access/transam/xlog.c         |  2 ++
 src/backend/access/transam/xlogreader.c   | 18 ------------------
 src/backend/replication/logical/logical.c |  2 ++
 src/bin/pg_rewind/parsexlog.c             |  6 ++++++
 src/bin/pg_waldump/pg_waldump.c           |  2 ++
 6 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 929da4eef2..9ec88b35ef 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 96e2115aad..425f7a12bd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6351,6 +6351,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7722,6 +7723,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 66bd9eb8d7..26f6834b8f 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -79,27 +79,11 @@ XLogReaderAllocate(int wal_segment_size)
 
 	state->max_block_id = -1;
 
-	/*
-	 * Permanently allocate readBuf.  We do it this way, rather than just
-	 * making a static array, for two reasons: (1) no need to waste the
-	 * storage in most instantiations of the backend; (2) a static char array
-	 * isn't guaranteed to have any particular alignment, whereas
-	 * palloc_extended() will provide MAXALIGN'd storage.
-	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
-											  MCXT_ALLOC_NO_OOM);
-	if (!state->readBuf)
-	{
-		pfree(state);
-		return NULL;
-	}
-
 	state->wal_segment_size = wal_segment_size;
 	state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -112,7 +96,6 @@ XLogReaderAllocate(int wal_segment_size)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -136,7 +119,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 56e2f8b0b0..bdf3c81b03 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1190,6 +1191,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Fri_Sep_06_16_33_18_2019_198)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v8 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9cab82e76d..80434aed2e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Fri_Sep_27_12_07_26_2019_308)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v6 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     |  2 ++
 src/backend/access/transam/xlog.c         |  2 ++
 src/backend/access/transam/xlogreader.c   | 18 ------------------
 src/backend/replication/logical/logical.c |  2 ++
 src/bin/pg_rewind/parsexlog.c             |  6 ++++++
 src/bin/pg_waldump/pg_waldump.c           |  2 ++
 6 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index a736504d62..358951c232 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1392,6 +1392,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1421,6 +1422,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c20d3b1418..a199600399 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index e00fa270ae..1f98dd0168 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -80,27 +80,11 @@ XLogReaderAllocate(int wal_segment_size)
 
 	state->max_block_id = -1;
 
-	/*
-	 * Permanently allocate readBuf.  We do it this way, rather than just
-	 * making a static array, for two reasons: (1) no need to waste the
-	 * storage in most instantiations of the backend; (2) a static char array
-	 * isn't guaranteed to have any particular alignment, whereas
-	 * palloc_extended() will provide MAXALIGN'd storage.
-	 */
-	state->readBuf = (char *) palloc_extended(XLOG_BLCKSZ,
-											  MCXT_ALLOC_NO_OOM);
-	if (!state->readBuf)
-	{
-		pfree(state);
-		return NULL;
-	}
-
 	state->wal_segment_size = wal_segment_size;
 	state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -113,7 +97,6 @@ XLogReaderAllocate(int wal_segment_size)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -137,7 +120,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 11e52e4c01..ea027caa69 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index ff26b30f82..c60267e87e 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index a61a5a91cb..5eba802720 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1108,6 +1108,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1188,6 +1189,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Tue_Sep_10_17_40_54_2019_177)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v13 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f5f6278880..6f5e6e5e56 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ce62559f42..e319793404 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6366,6 +6366,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7753,6 +7754,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index ad85f50c77..6b9287d68a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d0a002fae7..9f67674a88 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -516,6 +517,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 55337b65f1..48502151a1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -86,6 +87,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -109,6 +111,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -128,6 +131,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -169,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -218,6 +223,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index e82f184de2..a40f1d71e4 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1032,6 +1032,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1108,6 +1109,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Jan_28_21_20_20_2020_871)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v11 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b6c7125aa1..e69b6abae7 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9acc8d03e2..cb47aa239a 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index c7c05edcda..5a40047733 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1035,6 +1035,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1115,6 +1116,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Wed_Nov_27_12_09_23_2019_240)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v7 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index ced11bbdae..9e80a2cdb1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1391,6 +1391,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1420,6 +1421,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 49e8ca486e..4cbb6de1bb 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6349,6 +6349,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7721,6 +7722,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9932ba3882..b8ad4ffcd7 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -105,7 +105,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -118,7 +117,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -142,7 +140,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index db3e8f9dc0..99cb9bcc54 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -178,6 +178,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -523,6 +524,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index c5e5d75a87..d0e408c0a8 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -60,6 +60,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -92,6 +93,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -115,6 +117,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -222,6 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ee49712830..e17e66e688 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1094,6 +1094,7 @@ main(int argc, char **argv)
 	xlogreader_state = XLogReaderAllocate(WalSegSz, waldir);
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1174,6 +1175,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.16.3


----Next_Part(Wed_Sep_25_15_50_32_2019_576)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v15 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index b0d60a0d0f..5955bab699 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 06e7ff4a24..04478e78bc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6480,6 +6480,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7892,6 +7893,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 9281b57379..388bddaf41 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -112,7 +112,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -125,7 +124,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -152,7 +150,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index a15b0b3355..31f1029000 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 3dd6df4be6..26cba673f6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.4


----Next_Part(Thu_Jul__2_13_53_30_2020_072)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v10 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 8630ad2032..7f394156bd 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1337,6 +1337,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1367,6 +1368,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 694307a177..b144e3e241 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6477,6 +6477,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7889,6 +7890,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index bdde75086c..71a4931131 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -110,7 +110,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -123,7 +122,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -150,7 +148,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d99474c173..18d3248192 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -175,6 +175,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->page_read = page_read;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -525,6 +526,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index cad8e13965..c6247344d6 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir, NULL);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 02e71138cc..14f2eb1d28 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1052,6 +1052,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1131,6 +1132,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_May_26_16_40_02_2020_373)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v9 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 86b6540057..78073f969d 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -114,6 +116,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -133,6 +136,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -174,6 +178,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -224,6 +229,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Wed_Apr_22_10_12_46_2020_740)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v8 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 24b08810ac..8a8af68a39 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, lsn);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -1366,6 +1367,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 1ef6574f5f..080c546400 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6463,6 +6463,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7865,6 +7866,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 7c8c6fc314..aa8c463f46 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -146,7 +144,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7782e3ad2f..858c537558 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -518,6 +519,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 5b16681a37..c8b6c5ee90 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -61,6 +61,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, startpoint);
 	do
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	XLogBeginRead(xlogreader, ptr);
 	while (XLogReadRecord(xlogreader, &record, &errormsg) ==
@@ -132,6 +135,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -173,6 +177,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -223,6 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 1c42050277..451c10c976 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1040,6 +1040,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1119,6 +1120,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.18.2


----Next_Part(Tue_Apr_21_17_04_27_2020_275)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v12 4/4] Change policy of XLog read-buffer allocation
@ 2019-09-05 12:29  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Kyotaro Horiguchi @ 2019-09-05 12:29 UTC (permalink / raw)

Page buffer in XLogReaderState was allocated by XLogReaderAllcoate but
actually it'd be the responsibility to the callers of XLogReadRecord,
which now actually reads in pages. This patch does that.
---
 src/backend/access/transam/twophase.c     | 2 ++
 src/backend/access/transam/xlog.c         | 2 ++
 src/backend/access/transam/xlogreader.c   | 3 ---
 src/backend/replication/logical/logical.c | 2 ++
 src/bin/pg_rewind/parsexlog.c             | 6 ++++++
 src/bin/pg_waldump/pg_waldump.c           | 2 ++
 6 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f519a5b6a2..e72d5c7ccb 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -1336,6 +1336,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, lsn, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -1365,6 +1366,7 @@ XlogReadTwoPhaseData(XLogRecPtr lsn, char **buf, int *len)
 	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
 	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
 
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 }
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 364b1948e4..df1e9a5830 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -6352,6 +6352,7 @@ StartupXLOG(void)
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory"),
 				 errdetail("Failed while allocating a WAL reading processor.")));
+	xlogreader->readBuf = palloc(XLOG_BLCKSZ);
 	xlogreader->system_identifier = ControlFile->system_identifier;
 
 	/*
@@ -7736,6 +7737,7 @@ StartupXLOG(void)
 		close(readFile);
 		readFile = -1;
 	}
+	pfree(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 
 	/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 71984f00f7..81a95623a2 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -106,7 +106,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 										  MCXT_ALLOC_NO_OOM);
 	if (!state->errormsg_buf)
 	{
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -119,7 +118,6 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir)
 	if (!allocate_recordbuf(state, 0))
 	{
 		pfree(state->errormsg_buf);
-		pfree(state->readBuf);
 		pfree(state);
 		return NULL;
 	}
@@ -143,7 +141,6 @@ XLogReaderFree(XLogReaderState *state)
 	pfree(state->errormsg_buf);
 	if (state->readRecordBuf)
 		pfree(state->readRecordBuf);
-	pfree(state->readBuf);
 	pfree(state);
 }
 
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index d3a92044b5..59671f99f0 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -174,6 +174,7 @@ StartupDecodingContext(List *output_plugin_options,
 		ereport(ERROR,
 				(errcode(ERRCODE_OUT_OF_MEMORY),
 				 errmsg("out of memory")));
+	ctx->reader->readBuf = palloc(XLOG_BLCKSZ);
 	ctx->read_page = read_page;
 
 	ctx->reorder = ReorderBufferAllocate();
@@ -519,6 +520,7 @@ FreeDecodingContext(LogicalDecodingContext *ctx)
 
 	ReorderBufferFree(ctx->reorder);
 	FreeSnapshotBuilder(ctx->snapshot_builder);
+	pfree(ctx->reader->readBuf);
 	XLogReaderFree(ctx->reader);
 	MemoryContextDelete(ctx->context);
 }
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 4236ea1d12..ee380b25e1 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -58,6 +58,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	do
 	{
@@ -90,6 +91,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
 
 	} while (xlogreader->ReadRecPtr != endpoint);
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -113,6 +115,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	while (XLogReadRecord(xlogreader, ptr, &record, &errormsg) ==
 		   XLREAD_NEED_DATA)
@@ -131,6 +134,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
 	}
 	endptr = xlogreader->EndRecPtr;
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
@@ -172,6 +176,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 	xlogreader = XLogReaderAllocate(WalSegSz, datadir);
 	if (xlogreader == NULL)
 		pg_fatal("out of memory");
+	xlogreader->readBuf = pg_malloc(XLOG_BLCKSZ);
 
 	searchptr = forkptr;
 	for (;;)
@@ -220,6 +225,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
 		searchptr = record->xl_prev;
 	}
 
+	pg_free(xlogreader->readBuf);
 	XLogReaderFree(xlogreader);
 	if (xlogreadfd != -1)
 	{
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 16ee68efbe..4a6491007f 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -1036,6 +1036,7 @@ main(int argc, char **argv)
 
 	if (!xlogreader_state)
 		fatal_error("out of memory");
+	xlogreader_state->readBuf = palloc(XLOG_BLCKSZ);
 
 	/* first find a valid recptr to start from */
 	first_record = XLogFindNextRecord(xlogreader_state, private.startptr,
@@ -1116,6 +1117,7 @@ main(int argc, char **argv)
 					(uint32) xlogreader_state->ReadRecPtr,
 					errormsg);
 
+	pfree(xlogreader_state->readBuf);
 	XLogReaderFree(xlogreader_state);
 
 	return EXIT_SUCCESS;
-- 
2.23.0


----Next_Part(Fri_Nov_29_17_14_21_2019_330)----





^ permalink  raw  reply  [nested|flat] 14+ messages in thread

* [PATCH v20240912 2/4] Continue to remove some unnecesary strlen calls
@ 2024-09-11 04:21  Andy Fan <[email protected]>
  0 siblings, 0 replies; 14+ messages in thread

From: Andy Fan @ 2024-09-11 04:21 UTC (permalink / raw)

sprintf return the number of characters printed (not including the
trailing `\0'), so it is exactly same as strlen. so we can reuse that
value and avoid a strlen call.
---
 src/backend/utils/adt/datetime.c | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/backend/utils/adt/datetime.c b/src/backend/utils/adt/datetime.c
index 7abdc62f41..586bec8466 100644
--- a/src/backend/utils/adt/datetime.c
+++ b/src/backend/utils/adt/datetime.c
@@ -4594,6 +4594,7 @@ EncodeInterval(struct pg_itm *itm, int style, char *str)
 	int			fsec = itm->tm_usec;
 	bool		is_before = false;
 	bool		is_zero = true;
+	int			data_len;
 
 	/*
 	 * The sign of year and month are guaranteed to match, since they are
@@ -4651,11 +4652,11 @@ EncodeInterval(struct pg_itm *itm, int style, char *str)
 					char		sec_sign = (hour < 0 || min < 0 ||
 											sec < 0 || fsec < 0) ? '-' : '+';
 
-					sprintf(cp, "%c%d-%d %c%lld %c%lld:%02d:",
-							year_sign, abs(year), abs(mon),
-							day_sign, (long long) i64abs(mday),
-							sec_sign, (long long) i64abs(hour), abs(min));
-					cp += strlen(cp);
+					data_len = sprintf(cp, "%c%d-%d %c%lld %c%lld:%02d:",
+									   year_sign, abs(year), abs(mon),
+									   day_sign, (long long) i64abs(mday),
+									   sec_sign, (long long) i64abs(hour), abs(min));
+					cp += data_len;
 					cp = AppendSeconds(cp, sec, fsec, MAX_INTERVAL_PRECISION, true);
 					*cp = '\0';
 				}
@@ -4665,16 +4666,16 @@ EncodeInterval(struct pg_itm *itm, int style, char *str)
 				}
 				else if (has_day)
 				{
-					sprintf(cp, "%lld %lld:%02d:",
-							(long long) mday, (long long) hour, min);
-					cp += strlen(cp);
+					data_len = sprintf(cp, "%lld %lld:%02d:",
+									   (long long) mday, (long long) hour, min);
+					cp += data_len;
 					cp = AppendSeconds(cp, sec, fsec, MAX_INTERVAL_PRECISION, true);
 					*cp = '\0';
 				}
 				else
 				{
-					sprintf(cp, "%lld:%02d:", (long long) hour, min);
-					cp += strlen(cp);
+					data_len = sprintf(cp, "%lld:%02d:", (long long) hour, min);
+					cp += data_len;
 					cp = AppendSeconds(cp, sec, fsec, MAX_INTERVAL_PRECISION, true);
 					*cp = '\0';
 				}
-- 
2.45.1


--=-=-=
Content-Type: text/x-diff
Content-Disposition: attachment;
 filename=v20240912-0004-Make-printtup-a-bit-faster-intermediate-st.patch



^ permalink  raw  reply  [nested|flat] 14+ messages in thread

end of thread, other threads:[~2024-09-11 04:21 UTC | newest]

Thread overview: 14+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-09-05 12:29 [PATCH v8 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v13 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v7 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v15 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v10 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v9 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v5 3/3] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v9 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v11 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v6 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v8 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v12 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2019-09-05 12:29 [PATCH v7 4/4] Change policy of XLog read-buffer allocation Kyotaro Horiguchi <[email protected]>
2024-09-11 04:21 [PATCH v20240912 2/4] Continue to remove some unnecesary strlen calls Andy Fan <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox