pg.ddx.io pgsql-hackers@postgresql.org mailing list archive help / color / mirror / Atom feed From: Andrey Lepikhov <a.lepikhov@postgrespro.ru>
To: Michael Paquier <michael@paquier.xyz>
Cc: Dmitry Dolgov <9erthalion6@gmail.com>
Cc: Heikki Linnakangas <hlinnaka@iki.fi>
Cc: a.lubennikova@postgrespro.ru, PostgreSQL-development <pgsql-hackers@postgresql.org>
Subject: Re: Reduce amount of WAL generated by CREATE INDEX for gist, gin and sp-gist
Date: Tue, 5 Feb 2019 08:21:09 +0500
Message-ID: <af9f13ea-21aa-ea84-97cb-1bbfa5f30f8e@postgrespro.ru> (raw )
In-Reply-To: <20190204050413.GI29064@paquier.xyz >
References: <a055f129-518e-3eac-1242-833d58edfddb@postgrespro.ru >
<b929d56e-1c1d-dd5a-e21b-1baf06f6bffe@iki.fi >
<18f0b868-6888-27eb-22cc-982709c3d0f7@postgrespro.ru >
<CA+q6zcUmB4i1jQUBc1YU6zppV+u4ADjc05M1f-S3UJGW_0A7Sw@mail.gmail.com >
<1cdb9c52-e8fb-124b-3949-d3be8a6a7451@postgrespro.ru >
<20190204050413.GI29064@paquier.xyz >
On 04.02.2019 10:04, Michael Paquier wrote:
> On Tue, Dec 18, 2018 at 10:41:48AM +0500, Andrey Lepikhov wrote:
>> Ok. It is used only for demonstration.
>
> The latest patch set needs a rebase, so moved to next CF, waiting on
> author as this got no reviews.
The new version in attachment.
> --
> Michael
>
--
Andrey Lepikhov
Postgres Professional
https://postgrespro.com
The Russian Postgres Company
Attachments: download all (5 files, .tar.gz)
0001-Relation-into-WAL-function.patch (text/x-patch, 2.5K) download
From afc270f37cd082fb6e9f4ad694ea4cb123d98062 Mon Sep 17 00:00:00 2001
From: "Andrey V. Lepikhov" <a.lepikhov@postgrespro.ru>
Date: Tue, 5 Feb 2019 08:11:47 +0500
Subject: [PATCH 1/4] Relation-into-WAL-function
---
src/backend/access/spgist/spgutils.c | 1 +
src/backend/access/transam/generic_xlog.c | 39 +++++++++++++++++++++++
src/include/access/generic_xlog.h | 3 ++
3 files changed, 43 insertions(+)
diff --git a/src/backend/access/spgist/spgutils.c b/src/backend/access/spgist/spgutils.c
index 8e63c1fad2..b782bc2338 100644
--- a/src/backend/access/spgist/spgutils.c
+++ b/src/backend/access/spgist/spgutils.c
@@ -550,6 +550,7 @@ SpGistInitBuffer(Buffer b, uint16 f)
{
Assert(BufferGetPageSize(b) == BLCKSZ);
SpGistInitPage(BufferGetPage(b), f);
+ MarkBufferDirty(b);
}
/*
diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c
index 5b00b7275b..643da48345 100644
--- a/src/backend/access/transam/generic_xlog.c
+++ b/src/backend/access/transam/generic_xlog.c
@@ -542,3 +542,42 @@ generic_mask(char *page, BlockNumber blkno)
mask_unused_space(page);
}
+
+/*
+ * Function to write generic xlog for every existing block of a relation.
+ * Caller is responsible for locking the relation exclusively.
+ */
+void
+generic_log_relation(Relation rel)
+{
+ BlockNumber blkno;
+ BlockNumber nblocks = RelationGetNumberOfBlocks(rel);
+
+ for (blkno = 0; blkno < nblocks; )
+ {
+ GenericXLogState *state;
+ Buffer buffer[MAX_GENERIC_XLOG_PAGES];
+ int i,
+ blocks_pack;
+
+ CHECK_FOR_INTERRUPTS();
+
+ blocks_pack = ((nblocks-blkno) < MAX_GENERIC_XLOG_PAGES) ?
+ (nblocks-blkno) : MAX_GENERIC_XLOG_PAGES;
+
+ state = GenericXLogStart(rel);
+
+ for (i = 0 ; i < blocks_pack; i++)
+ {
+ buffer[i] = ReadBuffer(rel, blkno);
+ LockBuffer(buffer[i], BUFFER_LOCK_EXCLUSIVE);
+ GenericXLogRegisterBuffer(state, buffer[i], GENERIC_XLOG_FULL_IMAGE);
+ blkno++;
+ }
+
+ GenericXLogFinish(state);
+
+ for (i = 0 ; i < blocks_pack; i++)
+ UnlockReleaseBuffer(buffer[i]);
+ }
+}
diff --git a/src/include/access/generic_xlog.h b/src/include/access/generic_xlog.h
index cb5b5b713a..e3bbf014cc 100644
--- a/src/include/access/generic_xlog.h
+++ b/src/include/access/generic_xlog.h
@@ -42,4 +42,7 @@ extern const char *generic_identify(uint8 info);
extern void generic_desc(StringInfo buf, XLogReaderState *record);
extern void generic_mask(char *pagedata, BlockNumber blkno);
+/* other utils */
+extern void generic_log_relation(Relation rel);
+
#endif /* GENERIC_XLOG_H */
--
2.17.1
0002-GIN-Optimal-WAL-Usage.patch (text/x-patch, 10.5K) download
From 25dc935da9d5502a06fb9dc1eea4912fa0f48be1 Mon Sep 17 00:00:00 2001
From: "Andrey V. Lepikhov" <a.lepikhov@postgrespro.ru>
Date: Tue, 5 Feb 2019 08:13:29 +0500
Subject: [PATCH 2/4] GIN-Optimal-WAL-Usage
---
src/backend/access/gin/ginbtree.c | 6 ++---
src/backend/access/gin/gindatapage.c | 10 ++++----
src/backend/access/gin/ginentrypage.c | 2 +-
src/backend/access/gin/gininsert.c | 30 ++++++++++--------------
src/backend/access/gin/ginutil.c | 4 ++--
src/backend/access/gin/ginvacuum.c | 2 +-
src/backend/access/gin/ginxlog.c | 33 ---------------------------
src/backend/access/rmgrdesc/gindesc.c | 6 -----
src/include/access/gin.h | 3 ++-
src/include/access/ginxlog.h | 2 --
10 files changed, 27 insertions(+), 71 deletions(-)
diff --git a/src/backend/access/gin/ginbtree.c b/src/backend/access/gin/ginbtree.c
index 533949e46a..9f82eef8c3 100644
--- a/src/backend/access/gin/ginbtree.c
+++ b/src/backend/access/gin/ginbtree.c
@@ -396,7 +396,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack,
/* It will fit, perform the insertion */
START_CRIT_SECTION();
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
XLogBeginInsert();
XLogRegisterBuffer(0, stack->buffer, REGBUF_STANDARD);
@@ -417,7 +417,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack,
MarkBufferDirty(childbuf);
}
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
XLogRecPtr recptr;
ginxlogInsert xlrec;
@@ -595,7 +595,7 @@ ginPlaceToPage(GinBtree btree, GinBtreeStack *stack,
}
/* write WAL record */
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
XLogRecPtr recptr;
diff --git a/src/backend/access/gin/gindatapage.c b/src/backend/access/gin/gindatapage.c
index 3ad8b76710..a73cf944b3 100644
--- a/src/backend/access/gin/gindatapage.c
+++ b/src/backend/access/gin/gindatapage.c
@@ -593,7 +593,7 @@ dataBeginPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack,
* Great, all the items fit on a single page. If needed, prepare data
* for a WAL record describing the changes we'll make.
*/
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
computeLeafRecompressWALData(leaf);
/*
@@ -630,6 +630,7 @@ dataBeginPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack,
* subsequent insertions will probably also go to the end. This packs
* the index somewhat tighter when appending to a table, which is very
* common.
+ *
*/
if (!btree->isBuild)
{
@@ -719,7 +720,7 @@ dataExecPlaceToPageLeaf(GinBtree btree, Buffer buf, GinBtreeStack *stack,
dataPlaceToPageLeafRecompress(buf, leaf);
/* If needed, register WAL data built by computeLeafRecompressWALData */
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
XLogRegisterBufData(0, leaf->walinfo, leaf->walinfolen);
}
@@ -1152,7 +1153,7 @@ dataExecPlaceToPageInternal(GinBtree btree, Buffer buf, GinBtreeStack *stack,
pitem = (PostingItem *) insertdata;
GinDataPageAddPostingItem(page, pitem, off);
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
/*
* This must be static, because it has to survive until XLogInsert,
@@ -1773,6 +1774,7 @@ createPostingTree(Relation index, ItemPointerData *items, uint32 nitems,
Pointer ptr;
int nrootitems;
int rootsize;
+ bool is_build = (buildStats != NULL);
/* Construct the new root page in memory first. */
tmppage = (Page) palloc(BLCKSZ);
@@ -1826,7 +1828,7 @@ createPostingTree(Relation index, ItemPointerData *items, uint32 nitems,
PageRestoreTempPage(tmppage, page);
MarkBufferDirty(buffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !is_build)
{
XLogRecPtr recptr;
ginxlogCreatePostingTree data;
diff --git a/src/backend/access/gin/ginentrypage.c b/src/backend/access/gin/ginentrypage.c
index 4889de2a4f..1f5ba33d51 100644
--- a/src/backend/access/gin/ginentrypage.c
+++ b/src/backend/access/gin/ginentrypage.c
@@ -571,7 +571,7 @@ entryExecPlaceToPage(GinBtree btree, Buffer buf, GinBtreeStack *stack,
elog(ERROR, "failed to add item to index page in \"%s\"",
RelationGetRelationName(btree->index));
- if (RelationNeedsWAL(btree->index))
+ if (RelationNeedsWAL(btree->index) && !btree->isBuild)
{
/*
* This must be static, because it has to survive until XLogInsert,
diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index 524ac5be8b..6307743afd 100644
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -17,6 +17,7 @@
#include "access/gin_private.h"
#include "access/ginxlog.h"
#include "access/xloginsert.h"
+#include "access/generic_xlog.h"
#include "catalog/index.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
@@ -194,6 +195,7 @@ ginEntryInsert(GinState *ginstate,
buildStats->nEntries++;
ginPrepareEntryScan(&btree, attnum, key, category, ginstate);
+ btree.isBuild = (buildStats != NULL);
stack = ginFindLeafPage(&btree, false, false, NULL);
page = BufferGetPage(stack->buffer);
@@ -346,23 +348,6 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo)
GinInitBuffer(RootBuffer, GIN_LEAF);
MarkBufferDirty(RootBuffer);
- if (RelationNeedsWAL(index))
- {
- XLogRecPtr recptr;
- Page page;
-
- XLogBeginInsert();
- XLogRegisterBuffer(0, MetaBuffer, REGBUF_WILL_INIT | REGBUF_STANDARD);
- XLogRegisterBuffer(1, RootBuffer, REGBUF_WILL_INIT);
-
- recptr = XLogInsert(RM_GIN_ID, XLOG_GIN_CREATE_INDEX);
-
- page = BufferGetPage(RootBuffer);
- PageSetLSN(page, recptr);
-
- page = BufferGetPage(MetaBuffer);
- PageSetLSN(page, recptr);
- }
UnlockReleaseBuffer(MetaBuffer);
UnlockReleaseBuffer(RootBuffer);
@@ -417,7 +402,16 @@ ginbuild(Relation heap, Relation index, IndexInfo *indexInfo)
* Update metapage stats
*/
buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index);
- ginUpdateStats(index, &buildstate.buildStats);
+ ginUpdateStats(index, &buildstate.buildStats, true);
+
+ /*
+ * Create generic wal records for all pages of relation, if necessary.
+ * It seems reasonable not to generate WAL, if we recieved interrupt
+ * signal.
+ */
+ CHECK_FOR_INTERRUPTS();
+ if (RelationNeedsWAL(index))
+ generic_log_relation(index);
/*
* Return statistics
diff --git a/src/backend/access/gin/ginutil.c b/src/backend/access/gin/ginutil.c
index afc20232ac..51b20bca6e 100644
--- a/src/backend/access/gin/ginutil.c
+++ b/src/backend/access/gin/ginutil.c
@@ -661,7 +661,7 @@ ginGetStats(Relation index, GinStatsData *stats)
* Note: nPendingPages and ginVersion are *not* copied over
*/
void
-ginUpdateStats(Relation index, const GinStatsData *stats)
+ginUpdateStats(Relation index, const GinStatsData *stats, bool is_build)
{
Buffer metabuffer;
Page metapage;
@@ -691,7 +691,7 @@ ginUpdateStats(Relation index, const GinStatsData *stats)
MarkBufferDirty(metabuffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !is_build)
{
XLogRecPtr recptr;
ginxlogUpdateMeta data;
diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c
index dfe885b101..b9a28d1863 100644
--- a/src/backend/access/gin/ginvacuum.c
+++ b/src/backend/access/gin/ginvacuum.c
@@ -759,7 +759,7 @@ ginvacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats)
/* Update the metapage with accurate page and entry counts */
idxStat.nTotalPages = npages;
- ginUpdateStats(info->index, &idxStat);
+ ginUpdateStats(info->index, &idxStat, false);
/* Finally, vacuum the FSM */
IndexFreeSpaceMapVacuum(info->index);
diff --git a/src/backend/access/gin/ginxlog.c b/src/backend/access/gin/ginxlog.c
index c467ffa346..b648af1ff6 100644
--- a/src/backend/access/gin/ginxlog.c
+++ b/src/backend/access/gin/ginxlog.c
@@ -40,36 +40,6 @@ ginRedoClearIncompleteSplit(XLogReaderState *record, uint8 block_id)
UnlockReleaseBuffer(buffer);
}
-static void
-ginRedoCreateIndex(XLogReaderState *record)
-{
- XLogRecPtr lsn = record->EndRecPtr;
- Buffer RootBuffer,
- MetaBuffer;
- Page page;
-
- MetaBuffer = XLogInitBufferForRedo(record, 0);
- Assert(BufferGetBlockNumber(MetaBuffer) == GIN_METAPAGE_BLKNO);
- page = (Page) BufferGetPage(MetaBuffer);
-
- GinInitMetabuffer(MetaBuffer);
-
- PageSetLSN(page, lsn);
- MarkBufferDirty(MetaBuffer);
-
- RootBuffer = XLogInitBufferForRedo(record, 1);
- Assert(BufferGetBlockNumber(RootBuffer) == GIN_ROOT_BLKNO);
- page = (Page) BufferGetPage(RootBuffer);
-
- GinInitBuffer(RootBuffer, GIN_LEAF);
-
- PageSetLSN(page, lsn);
- MarkBufferDirty(RootBuffer);
-
- UnlockReleaseBuffer(RootBuffer);
- UnlockReleaseBuffer(MetaBuffer);
-}
-
static void
ginRedoCreatePTree(XLogReaderState *record)
{
@@ -767,9 +737,6 @@ gin_redo(XLogReaderState *record)
oldCtx = MemoryContextSwitchTo(opCtx);
switch (info)
{
- case XLOG_GIN_CREATE_INDEX:
- ginRedoCreateIndex(record);
- break;
case XLOG_GIN_CREATE_PTREE:
ginRedoCreatePTree(record);
break;
diff --git a/src/backend/access/rmgrdesc/gindesc.c b/src/backend/access/rmgrdesc/gindesc.c
index ef30ce16b0..f3f4e1b214 100644
--- a/src/backend/access/rmgrdesc/gindesc.c
+++ b/src/backend/access/rmgrdesc/gindesc.c
@@ -78,9 +78,6 @@ gin_desc(StringInfo buf, XLogReaderState *record)
switch (info)
{
- case XLOG_GIN_CREATE_INDEX:
- /* no further information */
- break;
case XLOG_GIN_CREATE_PTREE:
/* no further information */
break;
@@ -188,9 +185,6 @@ gin_identify(uint8 info)
switch (info & ~XLR_INFO_MASK)
{
- case XLOG_GIN_CREATE_INDEX:
- id = "CREATE_INDEX";
- break;
case XLOG_GIN_CREATE_PTREE:
id = "CREATE_PTREE";
break;
diff --git a/src/include/access/gin.h b/src/include/access/gin.h
index 61fa697039..d559ffc703 100644
--- a/src/include/access/gin.h
+++ b/src/include/access/gin.h
@@ -71,6 +71,7 @@ extern int gin_pending_list_limit;
/* ginutil.c */
extern void ginGetStats(Relation index, GinStatsData *stats);
-extern void ginUpdateStats(Relation index, const GinStatsData *stats);
+extern void ginUpdateStats(Relation index,
+ const GinStatsData *stats, bool is_build);
#endif /* GIN_H */
diff --git a/src/include/access/ginxlog.h b/src/include/access/ginxlog.h
index 9bd4e0b9ba..2c5d743cac 100644
--- a/src/include/access/ginxlog.h
+++ b/src/include/access/ginxlog.h
@@ -16,8 +16,6 @@
#include "lib/stringinfo.h"
#include "storage/off.h"
-#define XLOG_GIN_CREATE_INDEX 0x00
-
#define XLOG_GIN_CREATE_PTREE 0x10
typedef struct ginxlogCreatePostingTree
--
2.17.1
0003-GIST-Optimal-WAL-Usage.patch (text/x-patch, 11.1K) download
From f032a0645d86c57e3b89a294dfb46cfad1119825 Mon Sep 17 00:00:00 2001
From: "Andrey V. Lepikhov" <a.lepikhov@postgrespro.ru>
Date: Tue, 5 Feb 2019 08:14:41 +0500
Subject: [PATCH 3/4] GIST-Optimal-WAL-Usage
---
src/backend/access/gist/gist.c | 47 ++++++++++++++++++--------
src/backend/access/gist/gistbuild.c | 32 ++++++++++--------
src/backend/access/gist/gistutil.c | 2 +-
src/backend/access/gist/gistxlog.c | 22 ------------
src/backend/access/rmgrdesc/gistdesc.c | 5 ---
src/include/access/gist_private.h | 7 ++--
src/include/access/gistxlog.h | 1 -
7 files changed, 56 insertions(+), 60 deletions(-)
diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c
index b75b3a8dac..710c1bfd3a 100644
--- a/src/backend/access/gist/gist.c
+++ b/src/backend/access/gist/gist.c
@@ -173,7 +173,7 @@ gistinsert(Relation r, Datum *values, bool *isnull,
values, isnull, true /* size is currently bogus */ );
itup->t_tid = *ht_ctid;
- gistdoinsert(r, itup, 0, giststate, heapRel);
+ gistdoinsert(r, itup, 0, giststate, heapRel, false);
/* cleanup */
MemoryContextSwitchTo(oldCxt);
@@ -220,7 +220,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
Buffer leftchildbuf,
List **splitinfo,
bool markfollowright,
- Relation heapRel)
+ Relation heapRel,
+ bool is_build)
{
BlockNumber blkno = BufferGetBlockNumber(buffer);
Page page = BufferGetPage(buffer);
@@ -459,7 +460,7 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
* insertion for that. NB: The number of pages and data segments
* specified here must match the calculations in gistXLogSplit()!
*/
- if (RelationNeedsWAL(rel))
+ if (RelationNeedsWAL(rel) && !is_build)
XLogEnsureRecordSpace(npage, 1 + npage * 2);
START_CRIT_SECTION();
@@ -480,18 +481,20 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
PageRestoreTempPage(dist->page, BufferGetPage(dist->buffer));
dist->page = BufferGetPage(dist->buffer);
- /* Write the WAL record */
- if (RelationNeedsWAL(rel))
+ /*
+ * Write the WAL record.
+ * Do not write XLog entry if the insertion is caused by
+ * index build process.
+ */
+ if (RelationNeedsWAL(rel) && !is_build)
recptr = gistXLogSplit(is_leaf,
- dist, oldrlink, oldnsn, leftchildbuf,
- markfollowright);
+ dist, oldrlink, oldnsn, leftchildbuf,
+ markfollowright);
else
recptr = gistGetFakeLSN(rel);
for (ptr = dist; ptr; ptr = ptr->next)
- {
PageSetLSN(ptr->page, recptr);
- }
/*
* Return the new child buffers to the caller.
@@ -545,7 +548,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
if (BufferIsValid(leftchildbuf))
MarkBufferDirty(leftchildbuf);
- if (RelationNeedsWAL(rel))
+
+ if (RelationNeedsWAL(rel) && !is_build)
{
OffsetNumber ndeloffs = 0,
deloffs[1];
@@ -568,6 +572,7 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
PageSetLSN(page, recptr);
}
+
if (newblkno)
*newblkno = blkno;
}
@@ -584,17 +589,28 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
* the full page image. There's a chicken-and-egg problem: if we updated
* the child pages first, we wouldn't know the recptr of the WAL record
* we're about to write.
+ *
+ * We use fakeLSNs for inserions caused by index build. And when it is
+ * finished, we write generic_xlog entry for each index page and update
+ * all LSNs. In order to keep NSNs less then LSNs after this update, we
+ * set NSN to InvalidXLogRecPtr, which is the smallest possible NSN.
*/
+
if (BufferIsValid(leftchildbuf))
{
Page leftpg = BufferGetPage(leftchildbuf);
+ XLogRecPtr fakerecptr = InvalidXLogRecPtr;
- GistPageSetNSN(leftpg, recptr);
- GistClearFollowRight(leftpg);
+ if (!is_build)
+ GistPageSetNSN(leftpg, recptr);
+ else
+ GistPageSetNSN(leftpg, fakerecptr);
+ GistClearFollowRight(leftpg);
PageSetLSN(leftpg, recptr);
}
+
END_CRIT_SECTION();
return is_split;
@@ -605,9 +621,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
* this routine assumes it is invoked in a short-lived memory context,
* so it does not bother releasing palloc'd allocations.
*/
-void
gistdoinsert(Relation r, IndexTuple itup, Size freespace,
- GISTSTATE *giststate, Relation heapRel)
+ GISTSTATE *giststate, Relation heapRel, bool is_build)
{
ItemId iid;
IndexTuple idxtuple;
@@ -620,6 +635,7 @@ gistdoinsert(Relation r, IndexTuple itup, Size freespace,
state.freespace = freespace;
state.r = r;
state.heapRel = heapRel;
+ state.is_build = is_build;
/* Start from the root */
firststack.blkno = GIST_ROOT_BLKNO;
@@ -1237,7 +1253,8 @@ gistinserttuples(GISTInsertState *state, GISTInsertStack *stack,
leftchild,
&splitinfo,
true,
- state->heapRel);
+ state->heapRel,
+ state->is_build);
/*
* Before recursing up in case the page was split, release locks on the
diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c
index bd142a3560..2e98e55187 100644
--- a/src/backend/access/gist/gistbuild.c
+++ b/src/backend/access/gist/gistbuild.c
@@ -17,6 +17,7 @@
#include <math.h>
#include "access/genam.h"
+#include "access/generic_xlog.h"
#include "access/gist_private.h"
#include "access/gistxlog.h"
#include "access/xloginsert.h"
@@ -180,18 +181,12 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo)
MarkBufferDirty(buffer);
- if (RelationNeedsWAL(index))
- {
- XLogRecPtr recptr;
-
- XLogBeginInsert();
- XLogRegisterBuffer(0, buffer, REGBUF_WILL_INIT);
-
- recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_CREATE_INDEX);
- PageSetLSN(page, recptr);
- }
- else
- PageSetLSN(page, gistGetFakeLSN(heap));
+ /*
+ * Do not write index pages to WAL unitl index build is finished.
+ * But we still need increasing LSNs on each page, so use FakeLSN,
+ * even for relations which eventually need WAL.
+ */
+ PageSetLSN(page, gistGetFakeLSN(heap));
UnlockReleaseBuffer(buffer);
@@ -224,6 +219,15 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo)
freeGISTstate(buildstate.giststate);
+ /*
+ * Create generic wal records for all pages of relation, if necessary.
+ * It seems reasonable not to generate WAL, if we recieved interrupt
+ * signal.
+ */
+ CHECK_FOR_INTERRUPTS();
+ if (RelationNeedsWAL(index))
+ generic_log_relation(index);
+
/*
* Return statistics
*/
@@ -486,7 +490,7 @@ gistBuildCallback(Relation index,
* locked, we call gistdoinsert directly.
*/
gistdoinsert(index, itup, buildstate->freespace,
- buildstate->giststate, buildstate->heaprel);
+ buildstate->giststate, buildstate->heaprel, true);
}
/* Update tuple count and total size. */
@@ -693,7 +697,7 @@ gistbufferinginserttuples(GISTBuildState *buildstate, Buffer buffer, int level,
InvalidBuffer,
&splitinfo,
false,
- buildstate->heaprel);
+ buildstate->heaprel, true);
/*
* If this is a root split, update the root path item kept in memory. This
diff --git a/src/backend/access/gist/gistutil.c b/src/backend/access/gist/gistutil.c
index 8d3dfad27b..f7088e2d94 100644
--- a/src/backend/access/gist/gistutil.c
+++ b/src/backend/access/gist/gistutil.c
@@ -949,6 +949,7 @@ gistproperty(Oid index_oid, int attno,
* Temporary and unlogged GiST indexes are not WAL-logged, but we need LSNs
* to detect concurrent page splits anyway. This function provides a fake
* sequence of LSNs for that purpose.
+ * Persistent relations are also not WAL-logged while we build index.
*/
XLogRecPtr
gistGetFakeLSN(Relation rel)
@@ -969,7 +970,6 @@ gistGetFakeLSN(Relation rel)
* Unlogged relations are accessible from other backends, and survive
* (clean) restarts. GetFakeLSNForUnloggedRel() handles that for us.
*/
- Assert(rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED);
return GetFakeLSNForUnloggedRel();
}
}
diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c
index 408bd5390a..66d5bb831f 100644
--- a/src/backend/access/gist/gistxlog.c
+++ b/src/backend/access/gist/gistxlog.c
@@ -489,25 +489,6 @@ gistRedoPageSplitRecord(XLogReaderState *record)
UnlockReleaseBuffer(firstbuffer);
}
-static void
-gistRedoCreateIndex(XLogReaderState *record)
-{
- XLogRecPtr lsn = record->EndRecPtr;
- Buffer buffer;
- Page page;
-
- buffer = XLogInitBufferForRedo(record, 0);
- Assert(BufferGetBlockNumber(buffer) == GIST_ROOT_BLKNO);
- page = (Page) BufferGetPage(buffer);
-
- GISTInitBuffer(buffer, F_LEAF);
-
- PageSetLSN(page, lsn);
-
- MarkBufferDirty(buffer);
- UnlockReleaseBuffer(buffer);
-}
-
void
gist_redo(XLogReaderState *record)
{
@@ -532,9 +513,6 @@ gist_redo(XLogReaderState *record)
case XLOG_GIST_PAGE_SPLIT:
gistRedoPageSplitRecord(record);
break;
- case XLOG_GIST_CREATE_INDEX:
- gistRedoCreateIndex(record);
- break;
default:
elog(PANIC, "gist_redo: unknown op code %u", info);
}
diff --git a/src/backend/access/rmgrdesc/gistdesc.c b/src/backend/access/rmgrdesc/gistdesc.c
index e468c9e15a..75f1ed747e 100644
--- a/src/backend/access/rmgrdesc/gistdesc.c
+++ b/src/backend/access/rmgrdesc/gistdesc.c
@@ -52,8 +52,6 @@ gist_desc(StringInfo buf, XLogReaderState *record)
case XLOG_GIST_PAGE_SPLIT:
out_gistxlogPageSplit(buf, (gistxlogPageSplit *) rec);
break;
- case XLOG_GIST_CREATE_INDEX:
- break;
}
}
@@ -73,9 +71,6 @@ gist_identify(uint8 info)
case XLOG_GIST_PAGE_SPLIT:
id = "PAGE_SPLIT";
break;
- case XLOG_GIST_CREATE_INDEX:
- id = "CREATE_INDEX";
- break;
}
return id;
diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h
index 3698942f9d..b1680bfd6d 100644
--- a/src/include/access/gist_private.h
+++ b/src/include/access/gist_private.h
@@ -242,6 +242,7 @@ typedef struct
Relation r;
Relation heapRel;
Size freespace; /* free space to be left */
+ bool is_build;
GISTInsertStack *stack;
} GISTInsertState;
@@ -391,7 +392,8 @@ extern void gistdoinsert(Relation r,
IndexTuple itup,
Size freespace,
GISTSTATE *GISTstate,
- Relation heapRel);
+ Relation heapRel,
+ bool is_build);
/* A List of these is returned from gistplacetopage() in *splitinfo */
typedef struct
@@ -407,7 +409,8 @@ extern bool gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
Buffer leftchildbuf,
List **splitinfo,
bool markleftchild,
- Relation heapRel);
+ Relation heapRel,
+ bool is_build);
extern SplitedPageLayout *gistSplit(Relation r, Page page, IndexTuple *itup,
int len, GISTSTATE *giststate);
diff --git a/src/include/access/gistxlog.h b/src/include/access/gistxlog.h
index 5117aabf1a..2ee80e244e 100644
--- a/src/include/access/gistxlog.h
+++ b/src/include/access/gistxlog.h
@@ -22,7 +22,6 @@
/* #define XLOG_GIST_NEW_ROOT 0x20 */ /* not used anymore */
#define XLOG_GIST_PAGE_SPLIT 0x30
/* #define XLOG_GIST_INSERT_COMPLETE 0x40 */ /* not used anymore */
-#define XLOG_GIST_CREATE_INDEX 0x50
/* #define XLOG_GIST_PAGE_DELETE 0x60 */ /* not used anymore */
/*
--
2.17.1
0004-SPGIST-Optimal-WAL-Usage.patch (text/x-patch, 6.3K) download
From 3dbe3d827173244e9bda928147f35e0929266215 Mon Sep 17 00:00:00 2001
From: "Andrey V. Lepikhov" <a.lepikhov@postgrespro.ru>
Date: Tue, 5 Feb 2019 08:15:06 +0500
Subject: [PATCH 4/4] SPGIST-Optimal-WAL-Usage
---
src/backend/access/rmgrdesc/spgdesc.c | 5 ----
src/backend/access/spgist/spgdoinsert.c | 12 ++++-----
src/backend/access/spgist/spginsert.c | 24 +++--------------
src/backend/access/spgist/spgxlog.c | 35 -------------------------
src/include/access/spgxlog.h | 1 -
5 files changed, 10 insertions(+), 67 deletions(-)
diff --git a/src/backend/access/rmgrdesc/spgdesc.c b/src/backend/access/rmgrdesc/spgdesc.c
index 37af31a764..40c1c8b3f9 100644
--- a/src/backend/access/rmgrdesc/spgdesc.c
+++ b/src/backend/access/rmgrdesc/spgdesc.c
@@ -24,8 +24,6 @@ spg_desc(StringInfo buf, XLogReaderState *record)
switch (info)
{
- case XLOG_SPGIST_CREATE_INDEX:
- break;
case XLOG_SPGIST_ADD_LEAF:
{
spgxlogAddLeaf *xlrec = (spgxlogAddLeaf *) rec;
@@ -88,9 +86,6 @@ spg_identify(uint8 info)
switch (info & ~XLR_INFO_MASK)
{
- case XLOG_SPGIST_CREATE_INDEX:
- id = "CREATE_INDEX";
- break;
case XLOG_SPGIST_ADD_LEAF:
id = "ADD_LEAF";
break;
diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c
index 0d07b8b291..c34c44cd8b 100644
--- a/src/backend/access/spgist/spgdoinsert.c
+++ b/src/backend/access/spgist/spgdoinsert.c
@@ -289,7 +289,7 @@ addLeafTuple(Relation index, SpGistState *state, SpGistLeafTuple leafTuple,
MarkBufferDirty(current->buffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
int flags;
@@ -516,7 +516,7 @@ moveLeafs(Relation index, SpGistState *state,
MarkBufferDirty(current->buffer);
MarkBufferDirty(nbuf);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
@@ -1334,7 +1334,7 @@ doPickSplit(Relation index, SpGistState *state,
saveCurrent.buffer = InvalidBuffer;
}
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
int flags;
@@ -1531,7 +1531,7 @@ spgAddNodeAction(Relation index, SpGistState *state,
MarkBufferDirty(current->buffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
@@ -1644,7 +1644,7 @@ spgAddNodeAction(Relation index, SpGistState *state,
MarkBufferDirty(saveCurrent.buffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
int flags;
@@ -1840,7 +1840,7 @@ spgSplitNodeAction(Relation index, SpGistState *state,
MarkBufferDirty(current->buffer);
- if (RelationNeedsWAL(index))
+ if (RelationNeedsWAL(index) && !state->isBuild)
{
XLogRecPtr recptr;
diff --git a/src/backend/access/spgist/spginsert.c b/src/backend/access/spgist/spginsert.c
index f428a15138..5da0183a9b 100644
--- a/src/backend/access/spgist/spginsert.c
+++ b/src/backend/access/spgist/spginsert.c
@@ -21,6 +21,7 @@
#include "access/spgxlog.h"
#include "access/xlog.h"
#include "access/xloginsert.h"
+#include "access/generic_xlog.h"
#include "catalog/index.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
@@ -104,26 +105,6 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo)
SpGistInitBuffer(nullbuffer, SPGIST_LEAF | SPGIST_NULLS);
MarkBufferDirty(nullbuffer);
- if (RelationNeedsWAL(index))
- {
- XLogRecPtr recptr;
-
- XLogBeginInsert();
-
- /*
- * Replay will re-initialize the pages, so don't take full pages
- * images. No other data to log.
- */
- XLogRegisterBuffer(0, metabuffer, REGBUF_WILL_INIT | REGBUF_STANDARD);
- XLogRegisterBuffer(1, rootbuffer, REGBUF_WILL_INIT | REGBUF_STANDARD);
- XLogRegisterBuffer(2, nullbuffer, REGBUF_WILL_INIT | REGBUF_STANDARD);
-
- recptr = XLogInsert(RM_SPGIST_ID, XLOG_SPGIST_CREATE_INDEX);
-
- PageSetLSN(BufferGetPage(metabuffer), recptr);
- PageSetLSN(BufferGetPage(rootbuffer), recptr);
- PageSetLSN(BufferGetPage(nullbuffer), recptr);
- }
END_CRIT_SECTION();
@@ -150,6 +131,9 @@ spgbuild(Relation heap, Relation index, IndexInfo *indexInfo)
SpGistUpdateMetaPage(index);
+ if (RelationNeedsWAL(index))
+ generic_log_relation(index);
+
result = (IndexBuildResult *) palloc0(sizeof(IndexBuildResult));
result->heap_tuples = reltuples;
result->index_tuples = buildstate.indtuples;
diff --git a/src/backend/access/spgist/spgxlog.c b/src/backend/access/spgist/spgxlog.c
index 71836ee8a5..ebe6ae8715 100644
--- a/src/backend/access/spgist/spgxlog.c
+++ b/src/backend/access/spgist/spgxlog.c
@@ -72,38 +72,6 @@ addOrReplaceTuple(Page page, Item tuple, int size, OffsetNumber offset)
size);
}
-static void
-spgRedoCreateIndex(XLogReaderState *record)
-{
- XLogRecPtr lsn = record->EndRecPtr;
- Buffer buffer;
- Page page;
-
- buffer = XLogInitBufferForRedo(record, 0);
- Assert(BufferGetBlockNumber(buffer) == SPGIST_METAPAGE_BLKNO);
- page = (Page) BufferGetPage(buffer);
- SpGistInitMetapage(page);
- PageSetLSN(page, lsn);
- MarkBufferDirty(buffer);
- UnlockReleaseBuffer(buffer);
-
- buffer = XLogInitBufferForRedo(record, 1);
- Assert(BufferGetBlockNumber(buffer) == SPGIST_ROOT_BLKNO);
- SpGistInitBuffer(buffer, SPGIST_LEAF);
- page = (Page) BufferGetPage(buffer);
- PageSetLSN(page, lsn);
- MarkBufferDirty(buffer);
- UnlockReleaseBuffer(buffer);
-
- buffer = XLogInitBufferForRedo(record, 2);
- Assert(BufferGetBlockNumber(buffer) == SPGIST_NULL_BLKNO);
- SpGistInitBuffer(buffer, SPGIST_LEAF | SPGIST_NULLS);
- page = (Page) BufferGetPage(buffer);
- PageSetLSN(page, lsn);
- MarkBufferDirty(buffer);
- UnlockReleaseBuffer(buffer);
-}
-
static void
spgRedoAddLeaf(XLogReaderState *record)
{
@@ -976,9 +944,6 @@ spg_redo(XLogReaderState *record)
oldCxt = MemoryContextSwitchTo(opCtx);
switch (info)
{
- case XLOG_SPGIST_CREATE_INDEX:
- spgRedoCreateIndex(record);
- break;
case XLOG_SPGIST_ADD_LEAF:
spgRedoAddLeaf(record);
break;
diff --git a/src/include/access/spgxlog.h b/src/include/access/spgxlog.h
index 6527fc9eb1..8199b3f250 100644
--- a/src/include/access/spgxlog.h
+++ b/src/include/access/spgxlog.h
@@ -18,7 +18,6 @@
#include "storage/off.h"
/* XLOG record types for SPGiST */
-#define XLOG_SPGIST_CREATE_INDEX 0x00
#define XLOG_SPGIST_ADD_LEAF 0x10
#define XLOG_SPGIST_MOVE_LEAFS 0x20
#define XLOG_SPGIST_ADD_NODE 0x30
--
2.17.1
download
view thread (17+ messages) latest in thread
Message-ID: <af9f13ea-21aa-ea84-97cb-1bbfa5f30f8e@postgrespro.ru>
Permalink: ../af9f13ea-21aa-ea84-97cb-1bbfa5f30f8e@postgrespro.ru/
Also on: postgresql.org/message-id/af9f13ea-21aa-ea84-97cb-1bbfa5f30f8e@postgrespro.ru
copy link · copy postgr.es · copy git am
reply Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: a.lepikhov@postgrespro.ru, michael@paquier.xyz, 9erthalion6@gmail.com, hlinnaka@iki.fi
Subject: Re: Reduce amount of WAL generated by CREATE INDEX for gist, gin and sp-gist
In-Reply-To: <af9f13ea-21aa-ea84-97cb-1bbfa5f30f8e@postgrespro.ru>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox