public inbox for [email protected]
help / color / mirror / Atom feed[PATCH] cfe-12-gist_over_cfe-11-persistent squash commit
3+ messages / 3 participants
[nested] [flat]
* [PATCH] cfe-12-gist_over_cfe-11-persistent squash commit
@ 2021-03-15 14:20 Bruce Momjian <[email protected]>
0 siblings, 0 replies; 3+ messages in thread
From: Bruce Momjian @ 2021-03-15 14:20 UTC (permalink / raw)
---
src/backend/access/gist/README | 10 ++++++++++
src/backend/access/gist/gist.c | 21 +++++++++++++++------
src/backend/access/gist/gistbuild.c | 9 ++++++---
src/backend/access/gist/gistvacuum.c | 18 ++++++++++++------
4 files changed, 43 insertions(+), 15 deletions(-)
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README
index 25cab0047b..6b1f84634f 100644
--- a/src/backend/access/gist/README
+++ b/src/backend/access/gist/README
@@ -461,6 +461,16 @@ value. The page is not recycled, until that XID is no longer visible to
anyone. That's much more conservative than necessary, but let's keep it
simple.
+GiST and Encryption
+-------------------
+
+GiST uses LSNs and NSNs for concurrency. This means that unlogged and
+temporary relations also need LSNs. GiST has a mechanism to assign LSNs
+to such relations, but sometimes uses fixed LSNs and or calls
+gistGetFakeLSN(), which can cause duplicate LSNs to be used. Therefore,
+when encryption is enabled and fake LSNs are needed, the GiST code calls
+LSNForEncryption(). See src/backend/crypto/README for more details.
+
Authors:
Teodor Sigaev <[email protected]>
diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c
index 0683f42c25..5ca5f17db3 100644
--- a/src/backend/access/gist/gist.c
+++ b/src/backend/access/gist/gist.c
@@ -501,13 +501,16 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
* we don't need to be able to detect concurrent splits yet.)
*/
if (is_build)
- recptr = GistBuildLSN;
+ recptr = !FileEncryptionEnabled ? GistBuildLSN :
+ LSNForEncryption(RelationIsPermanent(rel));
else
{
if (RelationNeedsWAL(rel))
recptr = gistXLogSplit(is_leaf,
dist, oldrlink, oldnsn, leftchildbuf,
markfollowright);
+ else if (FileEncryptionEnabled)
+ recptr = LSNForEncryption(RelationIsPermanent(rel));
else
recptr = gistGetFakeLSN(rel);
}
@@ -568,7 +571,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
MarkBufferDirty(leftchildbuf);
if (is_build)
- recptr = GistBuildLSN;
+ recptr = !FileEncryptionEnabled ? GistBuildLSN :
+ LSNForEncryption(RelationIsPermanent(rel));
else
{
if (RelationNeedsWAL(rel))
@@ -586,6 +590,8 @@ gistplacetopage(Relation rel, Size freespace, GISTSTATE *giststate,
deloffs, ndeloffs, itup, ntup,
leftchildbuf);
}
+ else if (FileEncryptionEnabled)
+ recptr = LSNForEncryption(RelationIsPermanent(rel));
else
recptr = gistGetFakeLSN(rel);
}
@@ -1665,6 +1671,8 @@ gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel)
if (ndeletable > 0)
{
+ XLogRecPtr recptr;
+
TransactionId latestRemovedXid = InvalidTransactionId;
if (XLogStandbyInfoActive() && RelationNeedsWAL(rel))
@@ -1690,16 +1698,17 @@ gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
{
- XLogRecPtr recptr;
-
recptr = gistXLogDelete(buffer,
deletable, ndeletable,
latestRemovedXid);
- PageSetLSN(page, recptr);
}
+ else if (FileEncryptionEnabled)
+ recptr = LSNForEncryption(RelationIsPermanent(rel));
else
- PageSetLSN(page, gistGetFakeLSN(rel));
+ recptr = gistGetFakeLSN(rel);
+
+ PageSetLSN(page, recptr);
END_CRIT_SECTION();
}
diff --git a/src/backend/access/gist/gistbuild.c b/src/backend/access/gist/gistbuild.c
index 1054f6f1f2..a8b82483e8 100644
--- a/src/backend/access/gist/gistbuild.c
+++ b/src/backend/access/gist/gistbuild.c
@@ -299,7 +299,8 @@ gistbuild(Relation heap, Relation index, IndexInfo *indexInfo)
GISTInitBuffer(buffer, F_LEAF);
MarkBufferDirty(buffer);
- PageSetLSN(page, GistBuildLSN);
+ PageSetLSN(page, !FileEncryptionEnabled ? GistBuildLSN :
+ LSNForEncryption(RelationIsPermanent(index)));
UnlockReleaseBuffer(buffer);
@@ -451,7 +452,8 @@ gist_indexsortbuild(GISTBuildState *state)
/* Write out the root */
RelationOpenSmgr(state->indexrel);
- PageSetLSN(pagestate->page, GistBuildLSN);
+ PageSetLSN(pagestate->page, !FileEncryptionEnabled ? GistBuildLSN :
+ LSNForEncryption(RelationIsPermanent(state->indexrel)));
PageSetChecksumInplace(pagestate->page, GIST_ROOT_BLKNO);
smgrwrite(state->indexrel->rd_smgr, MAIN_FORKNUM, GIST_ROOT_BLKNO,
pagestate->page, true);
@@ -573,7 +575,8 @@ gist_indexsortbuild_flush_ready_pages(GISTBuildState *state)
if (blkno != state->pages_written)
elog(ERROR, "unexpected block number to flush GiST sorting build");
- PageSetLSN(page, GistBuildLSN);
+ PageSetLSN(page, !FileEncryptionEnabled ? GistBuildLSN :
+ LSNForEncryption(RelationIsPermanent(state->indexrel)));
PageSetChecksumInplace(page, blkno);
smgrextend(state->indexrel->rd_smgr, MAIN_FORKNUM, blkno, page, true);
diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c
index 0663193531..f783edd627 100644
--- a/src/backend/access/gist/gistvacuum.c
+++ b/src/backend/access/gist/gistvacuum.c
@@ -174,6 +174,8 @@ gistvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
vstate.callback_state = callback_state;
if (RelationNeedsWAL(rel))
vstate.startNSN = GetInsertRecPtr();
+ else if (FileEncryptionEnabled)
+ vstate.startNSN = LSNForEncryption(RelationIsPermanent(rel));
else
vstate.startNSN = gistGetFakeLSN(rel);
@@ -353,6 +355,8 @@ restart:
*/
if (ntodelete > 0)
{
+ XLogRecPtr recptr;
+
START_CRIT_SECTION();
MarkBufferDirty(buffer);
@@ -361,16 +365,15 @@ restart:
GistMarkTuplesDeleted(page);
if (RelationNeedsWAL(rel))
- {
- XLogRecPtr recptr;
-
recptr = gistXLogUpdate(buffer,
todelete, ntodelete,
NULL, 0, InvalidBuffer);
- PageSetLSN(page, recptr);
- }
+ else if (FileEncryptionEnabled)
+ recptr = LSNForEncryption(RelationIsPermanent(rel));
else
- PageSetLSN(page, gistGetFakeLSN(rel));
+ recptr = gistGetFakeLSN(rel);
+
+ PageSetLSN(page, recptr);
END_CRIT_SECTION();
@@ -657,8 +660,11 @@ gistdeletepage(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (RelationNeedsWAL(info->index))
recptr = gistXLogPageDelete(leafBuffer, txid, parentBuffer, downlink);
+ else if (FileEncryptionEnabled)
+ recptr = LSNForEncryption(RelationIsPermanent(info->index));
else
recptr = gistGetFakeLSN(info->index);
+
PageSetLSN(parentPage, recptr);
PageSetLSN(leafPage, recptr);
--
2.20.1
--AhhlLboLdkugWU4S
Content-Type: application/gzip
Content-Disposition: attachment;
filename="cfe-13-rel_over_cfe-12-gist.diff.gz"
Content-Transfer-Encoding: base64
H4sICDZtT2AAA2NmZS0xMy1yZWxfb3Zlcl9jZmUtMTItZ2lzdC5kaWZmAOQ8aXfbRpKfpV/R
9mw8pHiBh0jKWTvRQckc61pROXayeXgNoEkiAgEGAC0rcfa3b1V1A8RJUk6yk33LZ/MAqqu7
jq6rCzr3vQXrWgNr2Dd6vb7QjE672+VHHbPbEYOpMNrDYWfQFkOrIyx25blsIpasPWCa9pr+
sY6mtffPAc1rduKvTAFAi59s7rJ/N/Dn1wv5s7kK3u6f8VC8Rix11j5kV9yH0Z02a2uvO9rr
7pA1tJ6m7U9Wxk/CDF+zH26P70/f/cjMqWi0uw1fOLr3Qfg6/e40ZnYQsuDnFQ/mzPQWCzvc
3280Gvvwww1922gZjuct4N12A+GHTZOlXp/gf5fV9lngmy2Dmw/CtVrcNEUQtBA1va0HAfhh
m9Xg1aBJNo0yVrZj0VAY1e7AqEL4Oayc3pZ8JuRMuKh2yaLmgi9bvnj07VDgdxiB4IMy9K4R
+kKojzX6MprX4IGnuIXgPVYrpjZYEr34EfEX4fsl6EOfuwFftD463kxxFcE7WXDD88IAgJfr
bwnwHG9MHnLA2ApCz4+5GIMPcos3/adl6LWu+IOY2o7I6EMevQQ3VlPhmoC84orHqgJvD3qk
D8WvNJ6lF4QLHoTCT3yNeZxjgqIG553CEPhYzGLwDipUlq7MCMczuQPflVCGOT5E8MiElukt
nyzbj3WkM9gwA43whe3aYVqOJfBLtayUkqutlJ8lhLUQi2a+CFJbNsUo2zWdlSWSO685z23v
w3hREXxanvOUPFlnyBKii4YkOEskzJNTtHFIo3hAxNV5lgSA7/QY8jFg5py7M2HVWfeow+RG
sj03qNSqddYbMEs4Ql5oVPf3LXs6ZQ0wfCHjrVIrZ5Te2rddS3xkVndgCjEdHHabTdPQepyb
R/0umGGt3+uhCd2AfR/4s2mGr79mjfZgUAcNwo+2xuCK4ZBFFItl+FS5Ew5HkhgtprrP9tgB
+/7y5kI/OzmejPTTu9Hx/Yh5vrx4f3I5uT2NrvrC9Hyrydj9XPhi6vmizh4Fc4WwJKJwbgdM
fBAue5zjG3d0B346bxagsAvuNAmstV/buwUZjVxSiLG7dLgpKgsRcpRxnY2vx/f6+c3d++tv
ruosWvI4uBX+grvCDSty9XVAhC8Q7Mnlzc2VfjW6P749vhjpJ5fvr2+qX8J0ONFEhKdzYT4E
q0V+spKRoId7Aex7MvhyvsZb39LxWnaFRSjqMB5XVgEd89lBla1nDP2VoLWB6dRhC+DVyqv0
FI23+K77rmeJJr5l5kxr4xYnamwBUJp5aPLDaXtgGaCZhxZ81Y60w8O0Zm5BJPVzCxBq6aHW
rUMgU6PPHqop3iTZhB4xJNZUiDzqbGL/ItgUXGSwBJA6uxhP7if3qJQHODAIIbghlttTVrED
nVS+SjIArV2GPnvDXpzDpldKB4hHLjcciKu+YheA4QQHXE6u2WsaBC/4ce75a/hKkSLC4qqo
LXt7wgkEw9kjsGvYFsF3x5cShlQ1Xgqu+ftLbzZZOnaI63UEn0bqLDXaApA68xzLh/39QN/c
AGI3R0xDcw6LBZOYHrHg/sPUcxwPdHY2DzPLKiQ+s6znkMwk7v3G3t6v+La3gXa8vZX4xt6z
qU8NyZOPt7fRn17acxgQYS+g7kKE5xDjADYCJtjf8hqQA2PIVbCrrCKBkAtfMvhOb3AB3htv
XfExlKqtjBsioDvSvsg5AB1ttEG3jq71cKDVO/3/8xutoXi+w4ZjOyqmBNq7mU4DEV6vFobw
GdhDx4MLQIy2VjN18Yf2j7S1Ng2p5UcwiQdXkxw4Dr7ljm1VQMMB1F0tov2i1hXj0H4E1DHU
l+puYtq2uvbbfqNkx32ztEB8FRmmJrdPNEk9JqPO7HC1hN/4noRM7kBpYnYiqUYU1TYRVMvT
wyQ5f5l9XCs24xm+1nZla62Mq0zZiz/NetfW+2grySxtaNI2hkW2AAIZw3lwveruoUmUqW+O
GBSUClL40Bh2esOuGDabhtW1IHgWxtHuQUqEbXukEkGiFe0d9uqQW+NHuxMZUZ1WhPk6gVbQ
VJJ1m6CRZAdkK9EORXK4WQp3AmFdhe403tJ4ZfmzLFYgktvPMajgCXdShOwapFa0Dhhm5yxY
+QLxBIzDlw8cpqgz7lqojCJGXKe7rhdi1QcQumFThvfHAWYklRe4Oe6EeRuCTRi7H8gqIKEX
hYRWq+zVK1RJoKFSQvOnT6SzG5CwF29SfJGEFSQcOTZfHY+vN+cdWaYlMhCUvn53c3O/PfnI
TVw0dp18ZCZNpCHpBWewRBlIbjaVf8joYIB6fTjsluu1PnVWwVz3BbeedEJWqumFpuJZ0cDv
094/Sn2fo7+x0u6gtVm1LdHVMmX9HRoKa5PW+cuEjIr0sp6EQy2DYBOs485KSKPrLJ3pbnUJ
2WJsoWXOACmH0DviR8P+YDAcNJuCC/g11Xp8q0PIIiv1B1lAqrFonUPcN/Q5wH2jI4TOIfsw
ISQGBGGQCaxP4NaDihWntg/+BS/U2cp2w26HufQzqKqgOOZcgfdQfvn3aAlpBnO4WsU2Y5UF
TejFJl2IB9XZL8L3sCIKcRKvsymH2GM3xciX3YullINT6gHBQtvgw2m33Wz2ukZv2hH8qLtd
PfL4yjUkD4tK0pU6gh9HqCIwQsfbugIF/aBPaUPjYCEl/r3S2MEPsHSkR3ljsdFYw6rQNC2g
KP4E47DBlCRmSo5I4EYRbzEt+ZUUji82OusVlKpaDludZNAfHqEM+kcdVQz1+aMUgqyZFsig
zt7B/fvV0hEMgnQpEnQL+Lm7NNAVff/99+zMi0qkLPQi38M4w/o086YsnAsylgxLql8p97Pj
1o7zh60ChCwrLcANEtxsCErF9lly27r/s8dohRswA6R2vql1zJ4YDNoa7PyOafZ6Wtvsb935
WWSl2z4LSI6h36Hiez/SNyP8/1B8v78bjeL6945l9+yYZxXcU4P/1aX23NntFnWRUEpLQUet
wbQvOofNZt8yzLbRn3a2F91z2HZQUwVJdvFwUG93wTDi5yEFMEYIO5qYT0w6uf9ubRgPHpVp
RLGq2C4Zz6jEn+39GlUDXjwqG2CEj4GO3p/iXZJU4S32hlFkXQX0GEVplZPL0/eTf8oCDRhT
y3P/HrJAQLCulIphpRSAG4hDGlFpPouhAXgd+hePTXufx2TMW2rBorQlRRWlSTpyEzDVamuo
WEcLmbCGixX3N3InvyPUS1GR3LXrOH+3dAAAWwdbN0NBY0KhWubh1IbQTN7vtHuHmmg2hxYf
dPvcaptbN0QBvtItUQArjfeQgnr4GOKWgPvbjLd04DZo1kfAaLszan8BtksbjM4drLYvgqXn
WnQb0mjYUNKYU3YqEfkCeP4krI1G+rMN9OSWagO7H4/KmcqH7WqoCzHk7HWuLNGWZYn24Gg3
ScCGv7QfxKMdCNrYyHffg7yecja5sydLTLTHrh0izWkKL0fH5ypw+xPYvntRqF4y5JnszteA
NrB62CdWH2mfz2p3BYY0xIg5eDbb2afo1/U3l5eTP08KiP6ZUsgMeaYU1qM3S2GbRc30bhVa
tDSMsqTdQ25oHYh6exBaHGrDvtU1xPbUN4Or1Ipm4GSZXFbJqSayz/6mmnLYS+wQ9FzZuAK5
CA+5vgptJ2jOX6bAZIPQA/Zbpe+Ij8JchZ7fAosNfFuApBGilhsbNRelRi/swOQWxMGZ68uZ
DlSYIncZfSZepBppr92lImmv3ZPFnhPQ8Qn2xmGYXvng2XQASvHSqaQQq4AV5TRj6PeYMFLl
BreDzR37F3FC2XCi2KkGYQwVgk7bLoTxUwvrhI22jLBK9aW4ec/YDhN53oHRbQujMwB90bTu
Ubt72NOMcn0pxpXXl2I4ZG2HHC685/RFNRaCgEzPkZs8qytrkPBpmRWhUjhsHmiY3OX+0zPU
xbGN5c+t5c+BPQP276hLmDkErQW2Nq5ckzS7VFL5vklj8/0opTX4gB9xi0NKO5i2h1pHM46m
5RLK48lLJw9DO7nXBX9Qww8qXkWW9tRbPk0kZGVyNfNjBwEowfIlr1hBWF9XdPD4oEJA4DUw
b4hLxVFZsKqOyCl9QPM8Dr4Vvj21hTWigFxYyZA0eSh9O/5Wx4z5u+O76/H1BfgUvHI3ur25
u9exe0EeQG/HHK0tcTLrC2cp/ACcGCgKpClv2N3o8nZ0NwEbP7o+Henw/er4enR9nxylaNtl
YegbIKNfQnZWGd3d3dypLGCvInzfBK3Cq6c3ZyP97Pj+WD+9ubv75vZ+dFaN4BjALYJZ5aUt
TyhkOmO7jCo07IsVhql+JJYvgpeyNtbrt8la9zvS83+2iCVjV4HQH7kj6Uml2wBVkGNn1UAJ
IK4MbzoGyYoJvPwzxPT8kxCE3JT7ZPuajU13o73cgdSmo3WOLMhzjoaHvN8fDjYk/lksBTs5
A0HhHbkueCfHFXpL2bBj2Xje32y26N8+i6zYv1WSIFVCH+FrzhzP4A5y4+bkHxMY/1/Af2VA
PfjB9shxewiRRghoYnTxaslCNxcPW9ka938Xkhvd3Qd9o7ZetgD9ivioPFv8As9mDjSzDY6u
TUxuWeJDCwPYcobGMyBDtboG/Kxj+zm4rFrroPFHvfZr7AD/s3i+6ALuSOppgwiyik9taGBZ
ZJf25D8u2QXJhZ1hsc9bYmjELnxvtYzGE47xGaj++Hx8enw/vrnGS7BrNhGrBv6h5LWwgSYR
Z0Wd5uiakzdSPraWcszgp3xIpG136uWGpfrRXxbcoVB1SxyQuJdrPi+8O7XUQlqS098JBrZw
Xdi/nFxL6wZ5ElbL5EH0MkpYGgbofehhhmWbWBsAY8rBDj4IwoZn4488QJR0gqAKCJy5Hto5
TMEEN+cy5VI8/psFG9YV7OSbczB9Onicyfifo729dh+XibGtbbKVi+ENIKWkBEjU19Ux3f7w
Q3rwj9T7dDs7tZdz4Z+GH9kBhK5gnPHrG4a5zpf5+2cifT+eHGNmrM6p06D01JVNVcakk2UQ
TzqxZ9PtQI+5Knu1cJr92oZoW8buNWxMoxYAdkrq8F48sYMH8SQbvihsKG/A8kW48l0JCmOA
Wgz0L0QIWCrvry7u9Pej/9THZ+D2L1ULWZJ1GOIS03Qz/KhL8VZuL/TT8e270Z1+PJrop/d3
KbIradlBQgnzNt7CWxpMXnWEW42TTUVNvICq7ATzZir0wLjaxY4I21U8SxZNqbj1MXy5JiOW
8P8SGVF8sKZDrmAHOixRSMdvcusyJVzatzMbj1Fozz7a4VzWk6KgB2WMu0wqlxpGxY2E3pZr
ZmHlnDQwww7Zk5dnEi2rxhLBkey9RKbgtBCOBTr18SCk7vl6YC9sh/tKm90QA06wYkIprWp2
WSvlC7lhlZTRrGGJMi6xyI4iNucfsLUGA07stJFmDaxSY5kBXAUr7jhPNECiigLVCkoIll4l
DE3GKjdgtsRHU0g5YXvOhT25J3OoZgkExFcg6cdmVSJ75z2C5/Pj0zC1Js+FqRGH46h1TFMn
ANFJGk0Sep5EBgt/YhQDBiB1Lo8QkqacsGQpBLtseQQW8IWiEeEKuAEE0InwT6tAIgfrj8to
0ig59NgJICB+nGMwsx4eaVPAZjAMox1qcELan1hoLxIuxw4UNZ5FWU69YDVrdLB0kENqYStq
aCWEMIlERswy6OQRF42HjZKZWLVeL57kRWEXiR7epNyAHHBTyGzYwh5D7wmygizlGplqC8cC
IGSkCIUZSlxETICPCQS5g088JXqkFdKMmCjhcB7gdlG9X6CQj0oc8kBIPvCFoxYJrreUnidU
fM2dYO6tHAumowa4rNY/v9+xWmWfPrEXG7xWvOMoK8F4YN0kAJEAB44IE4waC9CsQaaD5Uac
5WbJf16JM2qx4XKRSiZYj6QC/PisDsKON6DcXA6fMYhCZLMc6KTcynxNJjE/UioQPDqT6ARN
4lC6rSKbOV/iOqego2qD4FZSzXZSkhKX54JdsLxHsHEgPkhTZyuI0/Cg5NH34N1WQ0HeCwqA
0WCklRPXIZEF3kKUKLnknpS+WmYYaRMARfHa+Nu6RCWVFUzNAugw0Ryhv4+1pczCsjfohiKJ
T6SY8IkK1e0Hsro6/v74cnxxXUHhedNKXnbrLlisnCtc8rZE03i7nhvkoxQgkm80uFI6GuUd
sFfsv2nY+eXxhX4yvr86nrynFWoRht20uZoIkgqVGtCxFyUMq0ZpfqpjNNcBWrxZ1MylYWSy
WFQeJcrihevYD8J5qrxYxzEK2dovQtK1KZ6pyAgy769Dv47GxXaXq5BFHT7yhbpxM024cuxK
TdyPxoKvnkEckhr7Svpwug9J1Mqh88ciyFxwL9fzIQ2VDviTq8BgoM60arU0yor6mmibfbF6
ua6dJAIMuV5UsSKyk7HYmfisWEwN+4vEYpsCLRU2pwOtP0uRVeC7nvdfqMjeKsTBf1lFVrz6
oxT51KPjKjPERUbHpQmNxnUnUmL48bty4k0JcbFeg1bnmCppvRILMMqVPMuZVs9UFuKwRRLl
BC6rDJnxFILLI8ksxMJcPlWWdfaqQpS8g1hK+HL/gEdaWrqDT3cqn4gQa98je+KXrPam5H48
u6x3yyoLq/RKVqD2kcKVYGZ+ouTNdGh2xf0HKvNAzBInJxjNjr/FOFrFbBAyOhRsRCAQpq4g
9fRVKIV/UMXhsCgZfICBc1B8Cw6IVq4NPhuCJewiHQcyXoGgyhAhIMDYbCW+Wgckpe6XVP1g
Wavhg4kfh1qaDojEIOf5aC9WC5j455WNQblaJCks2IcG2AeMxDpabxjF8VMMF23qcJGIsNcb
fvmYnQHbgzhW+q28vlv2dzmMXaBUadfodHtGd9g3u82m6JhWp2twbUO7UBm2fPW3DFI+AKDJ
/n+t7KgbK9EkhOKDSXtZfF1WN593VtmyHVXxzJ1h8lU4L7xB79EJd7vblZ1P3e6RPCi4jQm+
Al2sgCuD6H1m1pXZgO8ffvhRPtq2ACW+nd2LxRJLZEF0gL0uuv0hx957puMFInlvU+dE2V9w
MXaBis5nDrlhicN2u99sDru9fg+UrL/hQcMybHnFKoOkhwao8ak7yKlVpoJdcBCuPkvuRse7
u2tWSctFRvcMGw/X6bGHDefkSNpRFzdL7ajbjo6UuSX1QJfKnzltpM4a0rf02V6dncvjP0Oa
KHkSKJ8Ulp3o4Aqo65MM2Iz7BnolPGJGgxQ9EV52EKy6UIElJ/LRFUM5AellnzW04Dj5M8+T
99arYOuLux4v/yoH4drpcAwnPbnS/zm6u9FvrnWKgbAqgX2ousUXQIYle1irqk+jreEDgjX4
0q136O91nONTgVJ+FflxJgKTHeAfZkifEgPFblX1NbK9iDvgjmDYOwsr5XShoh59Vt6stOD+
q2zxpRQVHJgw565tUoERSIqZh07MsdanKhx72MBQoEiw5yCkwsLfA4XGe+BP4PXVkQoymj2q
ug+481CAX14FVH1IPMURIyRvpzwxEnjvUTMQkJgIDfEUr0C7VtPG25DPmgW6QqEZte9Ctp4k
LjEs0gv54GBu7sTZdmZ+BVeGCh8/j54QfwbWLFVJnLKBmNpd5QPzUVVGNWiD1C0RYAQCUc+E
ZPEoZMGN6kQB2AKITkh5PBl6ELLoiSIPQyS29D20lCJQFaP/qe5ae9tGruhn+1ewXrgrxZIs
6q0ku4Udy4lQ5wE/NigQgCBFMlbXFlVRStZo8997X8MZPiXHWaBdZG2ZHA7nPWfuPefKC9hg
hGeKW9zIPIEu88Tkeb+J15wVqXISaxt09XI1/8LSIFo+sYzwgB8lpb7H7VoGd/MHtVSyE65g
x3DmkbOe42toLk/fXV1fOtfTtxPnanLtvLq5vIQBUYNEMFBWa2F1dntDEqx3e6Mxr7Y0XdWU
5PmaViiK9gw5JzCZcK9qLVfBl3m0wVgMcDnCdZr8Jviq2e8vkgeydyD9X418lCJKSaLEovoV
rckWEiqxa7YpoorIIBRnTQ7HtJSUzaQSfRSrIM1HzU5QLy2gh5S8ONOFe3sGZTQlgtoZuRiR
5CoxhE6nGA8Aivv9UeACLLbbXcAvw9Gogl1Snt9W/GKkJTDZH5MEqi9Q8gJv84A7QWFH0S5f
sKFnTrnG5vcNf7DTVVqfdRJN6G8EIKEfhNbF6dnk9OY13AhhCi/WYS1ew4kT3nVwcWqdXFy8
f2XVDjeNQx/+1a1D/9PioJHI2HJ0pRZ037ssbUmViXZIm4Ab/mIh/Q+vNkyec+hbelQbwBDh
cD+IfiXC3RYmICaLlsGiZg7QFVOv3j6ccldOfUW1+h/eT7974qPGpWoDLZn1T5vz0E0n6HbC
Sd9qkXto62TPhIEsnm3pRCqYoDvyhjMvmLmtVtizu6MhHFoqKN4lmZXP8UxCGuodGunMGjQQ
/8vNAkCt37r91bwYP8THzKb+FUda4YFGVDg/gs+tiNtWnqeTRITk48gQydxHwwGvUnJTGPrh
KrqHv9Rxdx3RH2Jcm21WsVJqq8fwAWwq9GDK70TB/U2FSqNAPlfO9Opy8roWwok9XjsIxyXg
E+bl4MO53CTMUnkCgzdJrztfBcEZFOuPO6Td4ciklblDVbY7319nBZGiRUBcu30yA7PtsqnL
p3Pk0qksqTL7R7sllJfL4uDg4uBwBhaeZzApTvFnvPxh1dniDsM49EXc0u0A+jmyuz2muO5a
QpXVgkxZmHUUhg5cicS6n1zg2CjqcpMFMhRDDZAMNZWszjV0H7NHmuAhGtuCeB2TfbKpGFuv
3n/4h3N6c05mVTSiPrNEBpk4FKgOtK6R99eq5RvI+ps8ZT23dBaJhI+QriojHYupFkmRAH1P
1+L89IIFlIy8zbDb3EfxGtkRa1hp75H2wK3ckx2/rcfVTq38E6w081CKZd3EgWg/eU+gGC4A
EpHt4KKR0r1j07Xe7whw4+df5MlaqgFp3mRTpNqvnm4UlPUT/qTwsKKP4nCqo9EjK5eYI3hR
cpjh7Xx152uG6rWPJ9NrZ/IbbEcOFft8ejGBw/vJGWteefBBwRV1PvQbyVknX8/q9LlaF5YL
Va98kyJ6cYYvrXYhSZ1aZ8xxUMaDRqf9yObZM5jrDC4QDtA0IRrvp4PD+NPBc+vw/gDWN8mn
Xs+VD13WnKEHD//+IgVnPpIUPuCpaGKW5ITn4ahbrXEuwH2SvVmbBexKnwNfMlFoC4O3xWzU
R1s+HhvxcdJuzmBgkn+SuZu4QiY8A8mG3pjizCiIg7UpWOeOVMi8xPmjVdJq1eOIdOKY0i2S
iKeP9nZxsPP2YSTfQWxMz8Cpm9Y+6zj1ym8cNg66dxGh36FswFVMhI+X0+uJ7my0Qdd57az5
8doc21xriqUkH3eEXEkc7Qrso9KoaEPt2bA/8oejcas1Hve7ntcJ7AoFTnFeW/BWko6O8uRj
OOqKqwEOstDcNzJA1YEini7OPIrYids++295wvl8GTbzhXsP3YUWkGhZ15bS939HOtXPxINB
ghMaPYIVLvC0Erqz9QbGO86floqCQo5TOl910HMKt/AEf0g8OJSXIK986a5vGygQwQ8SZVEv
DgUJjioTZAAV/lIgR2roMNTZ2vNLNxURvbgb0omk78furN8feD0XgL3r9tudwbhtB9v7PpNZ
eednErISmoXQZf6A2/Vm6fjB2s2LKSUFxpssvrMLG35ncC2npcxtEnke3wf3fuBtPivoPUKB
6NFowKCMjOkw3k0XN6ET/Igf5M9Y06kYieJI5TIJDEW8uN8sMc5XOtBpWhAvClbd7RlkDu4L
sgeUs0z0ebT6zYRrtXt8b6mWe9NdTkpkaANAn4h+8ZKgVHZMsifPbhuN+4TG4FOOdK4TwjBD
VPYLzcZkMsohXeg3eakWtU8JAUfTK1hjZe0ptzTHcqagCfR+oXAuV9EXZuvdUqugZ5piLMww
YDTaidEQIfnAKnAXRfBs7CKxeBFtPt8SS5TIo1BPZu0KRlxCF7asCxd935sYmygKBeF2ho0u
NGuvazd6ScMWWAmqGpYX3TICRGIGZy69anJMUMsIxxNdGtFM+B4PW9PV8NTxWkaNEr6KEEgg
f3ybqfHAXWUqdnLsMSJlu4nFvMEUY7HgsbUX4IvfnC8SJx27gHjwvgu+Cn0SebN4jYI4X0en
qrowoKVadUORkZbbp2kPH5k/PFM8UoKHtI1YJKtB4IbsDhg/yNHAsw+pbOKNF+MpaSEEZbST
MZ8VBg9traGcM34mf0OAZOWVS44JGJYL9GawtdtoRc7KXa3chwaOZooOaJyBYKT8a0M2NzwA
kb5LUYlMLwRRoCUvtBMGAANIMuSS2uarywFKuJbQ1tpRQhTbBRLJqUIUScrgukJP6G4Wxho2
s9H3b4P7aPXwip0AbO+8jpapqw0DIBqEHMmjkQ2OYKBJU2Kh029bT9QoUE8oVhbzrAow7lPn
S9HyLlPm6YOZGy3bEttbgapsVjqzSP8fVNpkde5a6VIUmP2GG6/qruC+MJh1fLs96wxbrXa3
0x6H9rhX4UvJ5pIHfNkUJMcms2qvUxULoSTQAUz3+NiNH3KQLbm5hEOXu/ou9ke8nGdu/XO+
Pob/t/OKumObgsvA76HmFWGdK1hFDXF+/mUa30DzrzQVKQkwVsQt2oFcVMEu2o1eVPy9R175
PXVo9NqD/rDvj0cwgHqhbYdet4hKVJyHHj7F92nw2Iio6ScNHjEnIsH/3QdY52MCoLaN7QB7
YBPlXIRvSBjA2y5sCa6h3tGK0nxSVhAUP8BQXJXgnIL84MtrtvUSTUlMZ3dDHXRNpz2bXEyu
J2dJaptSp9QvXhAs+IuZAFTA0xShhQIZ9JlXZOb35uTKeX1yeXryeqKy7FGWJBWRiEXRQr8B
JSo+QLMUy4YoBiw9US9+CIjDbBHJl3QeAK9jxpJoI6IddQb7vRewyGuhlUrceKR4Sql1c2oM
WFvbf9jnCI0RGEMxUCpVQx+exShkpjTFdcQNiTCoudysYHVhxinzNeG2KK4EylB+ut+whHkp
CuMg4mr4otXZID8xI8w6fSifK9nv/PKq7u6q4Z/NRoNZ4I065Rr+sjcYGv7O6M+W8N8mFz5E
/NVieS2//Sgtf9X3qf1Zkv2f5uECfepC+n6TE5i/ScvwDX6ABHLK7zkqxJPQ5O82uMCbUmP5
GPOwI0Y0eYZTU2YLlKAAFcVEMhQl6XSky9K2zTqDJgwDCQCS0PM2DfmLdPLHq4IVWSWPpjJZ
P17kUpk1NCZ5YWStg85QnVroui7/Xj6v+n4Sk2zsDXvjcBi2WuHYHw+D4azdK98K8/nkp3g+
DfNQyBDSz2yJUDsSyr3BrbDQJAYJ4+ygZXI2UjfdHKE6eT02b8m9+TrImsbUrSgMEx72oMcB
KPtsvEG4hxNPJCTaboBrM5z+M1cI5ahn0resZ/rvF+n5k9ItiYNTNBb6+Ya19B3ZXer6+SL5
iyWBW61mPvu64A/ce5b4xRK4EyJ1DKY3757a6JOoUmGDQSCIL/S1THWeTEhfENCIAlbBekpt
50fWvykmDf1Xq+ExnrwH6B39cDJ13v82uSQfg/UcUIn1HzMxTB9kNKuk0yvnzeTkAyXEKjTN
9tPWtVTUIMiuuVdifTPSNXCNqsyu8rBDK1zlW9I+m6rMGroFAHlspxZLX1Ljd7uEP7sDCXt9
RdYN9krBgnWnxgQstDX1+VipbNiaWa/j7pa+ouLhHsgjCADZZeda96gTwyDtUZh57IDxvrlo
fo+N+cV+U7KgtfWJllS9iu+YW8FxPEVo3GHBLzE5p26bJVRNlvpeMyzSie9PYQUrKiZet3B5
Uy2IjWmi51RmkfFHo/DdlDX+eB2sUVSS3ueE3trrdxp9DDQ5atg2fbdCtmlxs6EY/u/VvDcL
nS3TIs2rpzoBFKVv6qJawR/KcS6v4iNrxgqctbsW7bi5cflYK7IeSboMu9t9k85/pCw2g0Ie
bT/T760curu9+VFGrKe82VIklRQ8YgCxEz7SX0NcjFmS+0k4+O64O/DavttqDQZjwEZ+f2Rv
x0c6n3J8pNOwT4OwBv408RE64M+ml4yPzHZ/NFtMr5/J8zswQzK9/USqWGEPJlUkClHT2u+0
Ou2Wvb//X1n/j02QfwAA
--AhhlLboLdkugWU4S--
^ permalink raw reply [nested|flat] 3+ messages in thread
* Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS
@ 2026-02-02 22:54 Radim Marek <[email protected]>
2026-02-03 10:57 ` Re: Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS Tomas Vondra <[email protected]>
0 siblings, 1 reply; 3+ messages in thread
From: Radim Marek @ 2026-02-02 22:54 UTC (permalink / raw)
To: pgsql-hackers
Hello,
First let me say this is a first time bug report and trying to patch the
PostgreSQL, fresh after the hacking webinar. Please let me know if I need
to clarify something.
TLDR; EXPLAIN ANALYZE BUFFERS reports different buffer counts for the Sort
node between first and subsequent executions per session.
Buffers: shared hit=4
-> Seq Scan on sort_buffer_test (cost=0.00..2.50 rows=1
width=32) (actual time=0.042..0.126 rows=1.00 loops=1)
Filter: (lower(email) = '[email protected]'::text)
Rows Removed by Filter: 99
Buffers: shared hit=1
vs
Buffers: shared hit=1
-> Seq Scan on sort_buffer_test (cost=0.00..2.50 rows=1
width=32) (actual time=0.021..0.057 rows=1.00 loops=1)
Filter: (lower(email) = '[email protected]'::text)
Rows Removed by Filter: 99
Buffers: shared hit=1
I found it while working on SQL regression tool (regresql) that uses total
reported buffers from EXPLAIN ANALYZE BUFFERS to detect deviations for
query performance.
The issue came on smaller queries where test cases showed consistent
"improvement" on performance (fewer buffers) when comparing baseline and
actual test runs. Previously did not notice it on larger data sets due to
the difference in buffers being negligible given the larger operations.
The example is the test case I managed to isolate.
--- way how to reproduce
CREATE TABLE sort_buffer_test (id serial PRIMARY KEY, val int);
INSERT INTO sort_buffer_test (val) SELECT generate_series(1, 100);
ANALYZE sort_buffer_test;
-- run twice in new psql session and observe reported buffer change
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF)
SELECT * FROM sort_buffer_test ORDER BY val DESC LIMIT 1;
---
This behaviour is session specific, tested on 17.x, 18.x and current
development version (just fiy, all verified on both amd64 and aarch64).
I traced the potential issue down in case of ORDER BY to pg_amop/pg_amproc
lookups. The specific lookups vary by operation but the pattern is the
same: first execution incurs catalog reads that get cached for subsequent
runs. This applies to DISTINCT (sort, not hashed), GROUP BY, window
functions, etc - which seems to isolate it to sort node.
All those I believe are actually metadata resolution (i.e. planner already
knows what to sort by).
Having said that - I'm happy to try to fix it. Just wondering what is the
right direction to go:
1. Pre-warm syscache during planning (tried locally and it works - but does
not feel conceptually right and it's rather 'hacky')
2. Pre-compute and store metadata and modify each plan node struct for each
type (lots of work). I believe that's what MergeJoin already does.
3. Or this is 'works as expected' - which I don't think it's the case and
would force me to do pre-warming on regresql side
Kind regards,
Radim
^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS
2026-02-02 22:54 Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS Radim Marek <[email protected]>
@ 2026-02-03 10:57 ` Tomas Vondra <[email protected]>
0 siblings, 0 replies; 3+ messages in thread
From: Tomas Vondra @ 2026-02-03 10:57 UTC (permalink / raw)
To: Radim Marek <[email protected]>; pgsql-hackers
On 2/2/26 23:54, Radim Marek wrote:
> Hello,
>
> First let me say this is a first time bug report and trying to patch the
> PostgreSQL, fresh after the hacking webinar. Please let me know if I
> need to clarify something.
>
> TLDR; EXPLAIN ANALYZE BUFFERS reports different buffer counts for the
> Sort node between first and subsequent executions per session.
>
> Buffers: shared hit=4
> -> Seq Scan on sort_buffer_test (cost=0.00..2.50 rows=1
> width=32) (actual time=0.042..0.126 rows=1.00 loops=1)
> Filter: (lower(email) = '[email protected]
> <mailto:[email protected]>'::text)
> Rows Removed by Filter: 99
> Buffers: shared hit=1
>
> vs
>
> Buffers: shared hit=1
> -> Seq Scan on sort_buffer_test (cost=0.00..2.50 rows=1
> width=32) (actual time=0.021..0.057 rows=1.00 loops=1)
> Filter: (lower(email) = '[email protected]
> <mailto:[email protected]>'::text)
> Rows Removed by Filter: 99
> Buffers: shared hit=1
>
> I found it while working on SQL regression tool (regresql) that uses
> total reported buffers from EXPLAIN ANALYZE BUFFERS to detect deviations
> for query performance.
>
> The issue came on smaller queries where test cases showed consistent
> "improvement" on performance (fewer buffers) when comparing baseline and
> actual test runs. Previously did not notice it on larger data sets due
> to the difference in buffers being negligible given the larger operations.
>
> The example is the test case I managed to isolate.
>
> --- way how to reproduce
>
> CREATE TABLE sort_buffer_test (id serial PRIMARY KEY, val int);
> INSERT INTO sort_buffer_test (val) SELECT generate_series(1, 100);
> ANALYZE sort_buffer_test;
>
> -- run twice in new psql session and observe reported buffer change
> EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF)
> SELECT * FROM sort_buffer_test ORDER BY val DESC LIMIT 1;
>
> ---
>
> This behaviour is session specific, tested on 17.x, 18.x and current
> development version (just fiy, all verified on both amd64 and aarch64).
>
> I traced the potential issue down in case of ORDER BY to pg_amop/
> pg_amproc lookups. The specific lookups vary by operation but the
> pattern is the same: first execution incurs catalog reads that get
> cached for subsequent runs. This applies to DISTINCT (sort, not hashed),
> GROUP BY, window functions, etc - which seems to isolate it to sort node.
>
> All those I believe are actually metadata resolution (i.e. planner
> already knows what to sort by).
>
Correct. AFAICS this happens because the execution may need additional
metadata that was not needed for planning. In this particular case the
planning checks that the operators are valid/compatible, but does not
lookup all the associated pg_amproc entries. That only happens during
execution, when actually constructing the "tuplesort" object.
But there are probably many similar cases.
> Having said that - I'm happy to try to fix it. Just wondering what is
> the right direction to go:
>
> 1. Pre-warm syscache during planning (tried locally and it works - but
> does not feel conceptually right and it's rather 'hacky')
Yeah, not going to happen. We'd have to preload the whole syscache,
because during planning we don't even know if that particular path/plan
will win. So we only do the bare minimum, to minimize the cost of
planning. And then we initialize the additional fields later, once we
have the cheapest plan.
> 2. Pre-compute and store metadata and modify each plan node struct for
> each type (lots of work). I believe that's what MergeJoin already does.
I suppose MergeJoin may simply need more metadata during planning. If it
does more than that, it might count as a bug (and it should do less).
> 3. Or this is 'works as expected' - which I don't think it's the case
> and would force me to do pre-warming on regresql side
>
I think it mostly works as expected/designed. That however does not mean
the issue you pointed out does not exist. IMHO what regresql is aiming
to do would be quite handy - e.g. in the index prefetching patch we have
regression tests checking that we're not accessing unexpected number of
buffers (or more buffers than master).
I'm planning to check if regresql would be a more elegant way to do
that. For now we're relying on plain regression tests (SQL + expected
output), and that happens to not have this issue because it executes the
queries in a session with a "deterministic" state.
regards
--
Tomas Vondra
^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2026-02-03 10:57 UTC | newest]
Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-15 14:20 [PATCH] cfe-12-gist_over_cfe-11-persistent squash commit Bruce Momjian <[email protected]>
2026-02-02 22:54 Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS Radim Marek <[email protected]>
2026-02-03 10:57 ` Re: Non-deterministic buffer counts reported in execution with EXPLAIN ANALYZE BUFFERS Tomas Vondra <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox