agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table 9+ messages / 2 participants [nested] [flat]
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 43 ++++++++++++++++------- src/backend/executor/nodeHashjoin.c | 22 ++++++++---- src/backend/utils/sort/sharedtuplestore.c | 8 +++++ src/include/executor/hashjoin.h | 30 ++++++++++------ src/include/executor/nodeHashjoin.h | 2 +- 5 files changed, 75 insertions(+), 30 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 5fd1c5553b..e529f50a3e 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or spillCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->spillCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashSpillContext", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,12 +574,19 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->spillCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldctx); + /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); @@ -913,7 +924,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) int oldnbatch = hashtable->nbatch; int curbatch = hashtable->curbatch; int nbatch; - MemoryContext oldcxt; long ninmemory; long nfreed; HashMemoryChunk oldchunks; @@ -934,13 +944,16 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); - if (hashtable->innerBatchFile == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); + /* we had no file arrays before */ hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldcxt); + /* time to establish the temp tablespaces, too */ PrepareTempTablespaces(); } @@ -951,8 +964,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable->outerBatchFile = repalloc0_array(hashtable->outerBatchFile, BufFile *, oldnbatch, nbatch); } - MemoryContextSwitchTo(oldcxt); - hashtable->nbatch = nbatch; /* @@ -1024,7 +1035,8 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) Assert(batchno > curbatch); ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1683,7 +1695,8 @@ ExecHashTableInsert(HashJoinTable hashtable, Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); } if (shouldFree) @@ -2664,7 +2677,8 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; @@ -3093,8 +3107,11 @@ ExecParallelHashJoinSetUpBatches(HashJoinTable hashtable, int nbatch) pstate->nbatch = nbatch; batches = dsa_get_address(hashtable->area, pstate->batches); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* + * Use hash join spill memory context to allocate accessors and their + * buffers + */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = nbatch; @@ -3196,8 +3213,8 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) */ Assert(DsaPointerIsValid(pstate->batches)); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* Use hash join spill memory context to allocate accessors */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = pstate->nbatch; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index a39164c15d..8618e0df40 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -491,7 +491,8 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->spillCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1313,21 +1314,30 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * If this is the first write to the batch file, this function first + * create it. The associated BufFile buffer is allocated in the given + * context. It is important to always give the HashSpillContext + * context. First to avoid a shorter-lived context, else the temp file + * buffers will get messed up. Second, for a better accounting of the + * spilling memory consumption. + * */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext cxt) { BufFile *file = *fileptr; if (file == NULL) { - /* First write to this batch file, so open it. */ + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(cxt); + file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index 0831249159..236be65f22 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -308,11 +308,15 @@ sts_puttuple(SharedTuplestoreAccessor *accessor, void *meta_data, { SharedTuplestoreParticipant *participant; char name[MAXPGPATH]; + MemoryContext oldcxt; /* Create one. Only this backend will write into it. */ sts_filename(name, accessor, accessor->participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->write_file = BufFileCreateFileSet(&accessor->fileset->fs, name); + MemoryContextSwitchTo(oldcxt); /* Set up the shared state for this backend's file. */ participant = &accessor->sts->participants[accessor->participant]; @@ -527,11 +531,15 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) if (accessor->read_file == NULL) { char name[MAXPGPATH]; + MemoryContext oldcxt; sts_filename(name, accessor, accessor->read_participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->read_file = BufFileOpenFileSet(&accessor->fileset->fs, name, O_RDONLY, false); + MemoryContextSwitchTo(oldcxt); } /* Seek and load the chunk header. */ diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index 8ee59d2c71..09113f0192 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -23,22 +23,31 @@ /* ---------------------------------------------------------------- * hash-join hash table structures * - * Each active hashjoin has a HashJoinTable control block, which is - * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. - * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * Each active hashjoin has a HashJoinTable structure, which is + * palloc'd in the executor's per-query context. Other storages needed for + * each hashjoin is kept in child contexts, three for each hashjoin: + * - HashTableContext (hashCxt): the parent hash table storage context + * - HashSpillContext (spillCxt): storage for temp files buffers + * - HashBatchContext (batchCxt): storage for a batch in serial hash join * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is * aborted early by an error. (Likewise, any temporary files we make will * be cleaned up by the virtual file manager in event of an error.) * - * Storage that should live through the entire join is allocated from the - * "hashCxt", while storage that is only wanted for the current batch is - * allocated in the "batchCxt". By resetting the batchCxt at the end of - * each batch, we free all the per-batch storage reliably and without tedium. + * The "hashCxt" context makes it easy and fast to release the storage when we + * don't need it anymore. Storage that should live through the entire join is + * allocated from the "hashCxt" (mainly hash's meta datas). + * + * Data associated with temp files lives in the "spillCxt" context which lives + * during the entire join as temp files might need to survives batches. These + * files are explicitly destroyed by calling BufFileClose() when the code is + * done with them. The aim of this context is to help accounting the memory + * allocations dedicated to temp files and their buffers. + * + * Finaly, storage that is only wanted for the current batch is allocated in + * the "batchCxt". By resetting the batchCxt at the end of each batch, we free + * all the per-batch storage reliably and without tedium. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -350,6 +359,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext spillCxt; /* context for spilling to temp files */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..958df42240 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext spillcxt); #endif /* NODEHASHJOIN_H */ -- 2.40.1 --MP_/WHuN2qnG.U=I.v0AxCf+Cxe-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 43 ++++++++++++++++------- src/backend/executor/nodeHashjoin.c | 18 +++++++--- src/backend/utils/sort/sharedtuplestore.c | 8 +++++ src/include/executor/hashjoin.h | 15 ++++++-- src/include/executor/nodeHashjoin.h | 2 +- 5 files changed, 64 insertions(+), 22 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 5fd1c5553b..a4fbf29301 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or fileCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->fileCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashBatchFiles", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,15 +574,21 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); + + MemoryContextSwitchTo(oldctx); } MemoryContextSwitchTo(oldcxt); @@ -913,7 +923,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) int oldnbatch = hashtable->nbatch; int curbatch = hashtable->curbatch; int nbatch; - MemoryContext oldcxt; long ninmemory; long nfreed; HashMemoryChunk oldchunks; @@ -934,13 +943,16 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); - if (hashtable->innerBatchFile == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); + /* we had no file arrays before */ hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldcxt); + /* time to establish the temp tablespaces, too */ PrepareTempTablespaces(); } @@ -951,8 +963,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable->outerBatchFile = repalloc0_array(hashtable->outerBatchFile, BufFile *, oldnbatch, nbatch); } - MemoryContextSwitchTo(oldcxt); - hashtable->nbatch = nbatch; /* @@ -1022,9 +1032,11 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) { /* dump it out */ Assert(batchno > curbatch); + ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1681,9 +1693,11 @@ ExecHashTableInsert(HashJoinTable hashtable, * put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); } if (shouldFree) @@ -2663,8 +2677,11 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) { /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); + pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; @@ -3093,8 +3110,8 @@ ExecParallelHashJoinSetUpBatches(HashJoinTable hashtable, int nbatch) pstate->nbatch = nbatch; batches = dsa_get_address(hashtable->area, pstate->batches); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* Use hash join file memory context. */ + oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = nbatch; @@ -3196,8 +3213,8 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) */ Assert(DsaPointerIsValid(pstate->batches)); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* Use hash join file memory context. */ + oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = pstate->nbatch; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 920d1831c2..ac72fbfbb6 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -485,8 +485,10 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) */ Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->fileCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1308,21 +1310,27 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * Note: it is important always to call this in the HashBatchFiles context, + * not in a shorter-lived context; else the temp file buffers will get messed + * up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext filecxt) { BufFile *file = *fileptr; if (file == NULL) { + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(filecxt); + /* First write to this batch file, so open it. */ file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index 0831249159..236be65f22 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -308,11 +308,15 @@ sts_puttuple(SharedTuplestoreAccessor *accessor, void *meta_data, { SharedTuplestoreParticipant *participant; char name[MAXPGPATH]; + MemoryContext oldcxt; /* Create one. Only this backend will write into it. */ sts_filename(name, accessor, accessor->participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->write_file = BufFileCreateFileSet(&accessor->fileset->fs, name); + MemoryContextSwitchTo(oldcxt); /* Set up the shared state for this backend's file. */ participant = &accessor->sts->participants[accessor->participant]; @@ -527,11 +531,15 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) if (accessor->read_file == NULL) { char name[MAXPGPATH]; + MemoryContext oldcxt; sts_filename(name, accessor, accessor->read_participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->read_file = BufFileOpenFileSet(&accessor->fileset->fs, name, O_RDONLY, false); + MemoryContextSwitchTo(oldcxt); } /* Seek and load the chunk header. */ diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index 8ee59d2c71..74867c3e40 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -25,10 +25,14 @@ * * Each active hashjoin has a HashJoinTable control block, which is * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. + * for the hashjoin is kept in private memory contexts, three for each + * hashjoin: + * - HashTableContext (hashCxt): the control block associated to the hash table + * - HashBatchContext (batchCxt): storages for batches + * - HashBatchFiles (fileCxt): storage for temp files buffers + * * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * anymore. * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is @@ -39,6 +43,10 @@ * "hashCxt", while storage that is only wanted for the current batch is * allocated in the "batchCxt". By resetting the batchCxt at the end of * each batch, we free all the per-batch storage reliably and without tedium. + * Note that data associated with the temp files lives in the "fileCxt" context + * which lives during the entire join as temp files might need to survives + * batches. These files are explicitly destroyed by calling BufFileClose() + * when the code is done with them. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -350,6 +358,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext fileCxt; /* context for the BufFile related storage */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..a8f9ae1989 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext filecxt); #endif /* NODEHASHJOIN_H */ -- 2.39.2 --MP_/w5PdZ2hoFAvEnG3QNPoMNWk-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 43 ++++++++++++++++------- src/backend/executor/nodeHashjoin.c | 24 +++++++++---- src/backend/utils/sort/sharedtuplestore.c | 8 +++++ src/include/executor/hashjoin.h | 30 ++++++++++------ src/include/executor/nodeHashjoin.h | 2 +- 5 files changed, 76 insertions(+), 31 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 5fd1c5553b..e529f50a3e 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or spillCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->spillCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashSpillContext", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,12 +574,19 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->spillCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldctx); + /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); @@ -913,7 +924,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) int oldnbatch = hashtable->nbatch; int curbatch = hashtable->curbatch; int nbatch; - MemoryContext oldcxt; long ninmemory; long nfreed; HashMemoryChunk oldchunks; @@ -934,13 +944,16 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); - if (hashtable->innerBatchFile == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); + /* we had no file arrays before */ hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldcxt); + /* time to establish the temp tablespaces, too */ PrepareTempTablespaces(); } @@ -951,8 +964,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable->outerBatchFile = repalloc0_array(hashtable->outerBatchFile, BufFile *, oldnbatch, nbatch); } - MemoryContextSwitchTo(oldcxt); - hashtable->nbatch = nbatch; /* @@ -1024,7 +1035,8 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) Assert(batchno > curbatch); ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1683,7 +1695,8 @@ ExecHashTableInsert(HashJoinTable hashtable, Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); } if (shouldFree) @@ -2664,7 +2677,8 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; @@ -3093,8 +3107,11 @@ ExecParallelHashJoinSetUpBatches(HashJoinTable hashtable, int nbatch) pstate->nbatch = nbatch; batches = dsa_get_address(hashtable->area, pstate->batches); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* + * Use hash join spill memory context to allocate accessors and their + * buffers + */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = nbatch; @@ -3196,8 +3213,8 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) */ Assert(DsaPointerIsValid(pstate->batches)); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* Use hash join spill memory context to allocate accessors */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = pstate->nbatch; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 920d1831c2..3b903cc905 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -486,7 +486,8 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->spillCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1307,22 +1308,31 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. - * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext spillcxt) { BufFile *file = *fileptr; if (file == NULL) { - /* First write to this batch file, so open it. */ + MemoryContext oldctx; + + /* + * First write to this batch file, so open it. + * + * It is important to always create BufFile'd temp files in the + * HashSpillContext context. First to avoid a shorter-lived context, + * else the temp file buffers will get messed up. Second, for a better + * accounting of the spilling memory consumption. + */ + oldctx = MemoryContextSwitchTo(spillcxt); + file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index 0831249159..236be65f22 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -308,11 +308,15 @@ sts_puttuple(SharedTuplestoreAccessor *accessor, void *meta_data, { SharedTuplestoreParticipant *participant; char name[MAXPGPATH]; + MemoryContext oldcxt; /* Create one. Only this backend will write into it. */ sts_filename(name, accessor, accessor->participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->write_file = BufFileCreateFileSet(&accessor->fileset->fs, name); + MemoryContextSwitchTo(oldcxt); /* Set up the shared state for this backend's file. */ participant = &accessor->sts->participants[accessor->participant]; @@ -527,11 +531,15 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) if (accessor->read_file == NULL) { char name[MAXPGPATH]; + MemoryContext oldcxt; sts_filename(name, accessor, accessor->read_participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->read_file = BufFileOpenFileSet(&accessor->fileset->fs, name, O_RDONLY, false); + MemoryContextSwitchTo(oldcxt); } /* Seek and load the chunk header. */ diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index 8ee59d2c71..09113f0192 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -23,22 +23,31 @@ /* ---------------------------------------------------------------- * hash-join hash table structures * - * Each active hashjoin has a HashJoinTable control block, which is - * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. - * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * Each active hashjoin has a HashJoinTable structure, which is + * palloc'd in the executor's per-query context. Other storages needed for + * each hashjoin is kept in child contexts, three for each hashjoin: + * - HashTableContext (hashCxt): the parent hash table storage context + * - HashSpillContext (spillCxt): storage for temp files buffers + * - HashBatchContext (batchCxt): storage for a batch in serial hash join * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is * aborted early by an error. (Likewise, any temporary files we make will * be cleaned up by the virtual file manager in event of an error.) * - * Storage that should live through the entire join is allocated from the - * "hashCxt", while storage that is only wanted for the current batch is - * allocated in the "batchCxt". By resetting the batchCxt at the end of - * each batch, we free all the per-batch storage reliably and without tedium. + * The "hashCxt" context makes it easy and fast to release the storage when we + * don't need it anymore. Storage that should live through the entire join is + * allocated from the "hashCxt" (mainly hash's meta datas). + * + * Data associated with temp files lives in the "spillCxt" context which lives + * during the entire join as temp files might need to survives batches. These + * files are explicitly destroyed by calling BufFileClose() when the code is + * done with them. The aim of this context is to help accounting the memory + * allocations dedicated to temp files and their buffers. + * + * Finaly, storage that is only wanted for the current batch is allocated in + * the "batchCxt". By resetting the batchCxt at the end of each batch, we free + * all the per-batch storage reliably and without tedium. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -350,6 +359,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext spillCxt; /* context for spilling to temp files */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..958df42240 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext spillcxt); #endif /* NODEHASHJOIN_H */ -- 2.39.2 --MP_/=afXY/P8/zXs58RNNJ5YG6H-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 27 ++++++++++++++++++++++----- src/backend/executor/nodeHashjoin.c | 18 +++++++++++++----- src/include/executor/hashjoin.h | 15 ++++++++++++--- src/include/executor/nodeHashjoin.h | 2 +- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 748c9b0024..0c0d5b4a3c 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or fileCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->fileCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashBatchFiles", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,15 +574,21 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); + + MemoryContextSwitchTo(oldctx); } MemoryContextSwitchTo(oldcxt); @@ -934,7 +944,7 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); if (hashtable->innerBatchFile == NULL) { @@ -1022,9 +1032,11 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) { /* dump it out */ Assert(batchno > curbatch); + ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1681,9 +1693,11 @@ ExecHashTableInsert(HashJoinTable hashtable, * put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); } if (shouldFree) @@ -2534,8 +2548,11 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) { /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); + pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index f189fb4d28..6055abde49 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -432,8 +432,10 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) */ Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->fileCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1234,21 +1236,27 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * Note: it is important always to call this in the HashBatchFiles context, + * not in a shorter-lived context; else the temp file buffers will get messed + * up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext filecxt) { BufFile *file = *fileptr; if (file == NULL) { + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(filecxt); + /* First write to this batch file, so open it. */ file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index acb7592ca0..d759235d7f 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -25,10 +25,14 @@ * * Each active hashjoin has a HashJoinTable control block, which is * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. + * for the hashjoin is kept in private memory contexts, three for each + * hashjoin: + * - HashTableContext (hashCxt): the control block associated to the hash table + * - HashBatchContext (batchCxt): storages for batches + * - HashBatchFiles (fileCxt): storage for temp files buffers + * * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * anymore. * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is @@ -39,6 +43,10 @@ * "hashCxt", while storage that is only wanted for the current batch is * allocated in the "batchCxt". By resetting the batchCxt at the end of * each batch, we free all the per-batch storage reliably and without tedium. + * Note that data associated with the temp files lives in the "fileCxt" context + * which lives during the entire join as temp files might need to survives + * batches. These files are explicitly destroyed by calling BufFileClose() + * when the code is done with them. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -348,6 +356,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext fileCxt; /* context for the BufFile related storage */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..a8f9ae1989 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext filecxt); #endif /* NODEHASHJOIN_H */ -- 2.39.2 --MP_/HNeUeUP7WKSzkiBqcPpwVN+-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH 2/4] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 35 +++++++++++++++++++++-------- src/backend/executor/nodeHashjoin.c | 18 ++++++++++----- src/include/executor/hashjoin.h | 15 ++++++++++--- src/include/executor/nodeHashjoin.h | 2 +- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index a45bd3a315..4544296391 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or fileCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->fileCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashBatchFiles", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,15 +574,21 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); + + MemoryContextSwitchTo(oldctx); } MemoryContextSwitchTo(oldcxt); @@ -913,7 +923,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) int oldnbatch = hashtable->nbatch; int curbatch = hashtable->curbatch; int nbatch; - MemoryContext oldcxt; long ninmemory; long nfreed; HashMemoryChunk oldchunks; @@ -934,13 +943,16 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); - if (hashtable->innerBatchFile == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); + /* we had no file arrays before */ hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldcxt); + /* time to establish the temp tablespaces, too */ PrepareTempTablespaces(); } @@ -951,8 +963,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable->outerBatchFile = repalloc0_array(hashtable->outerBatchFile, BufFile *, oldnbatch, nbatch); } - MemoryContextSwitchTo(oldcxt); - hashtable->nbatch = nbatch; /* @@ -1022,9 +1032,11 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) { /* dump it out */ Assert(batchno > curbatch); + ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1681,9 +1693,11 @@ ExecHashTableInsert(HashJoinTable hashtable, * put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); } if (shouldFree) @@ -2663,8 +2677,11 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) { /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); + pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 5454afbff2..5bc7f814c6 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -485,8 +485,10 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) */ Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->fileCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1297,21 +1299,27 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * Note: it is important always to call this in the HashBatchFiles context, + * not in a shorter-lived context; else the temp file buffers will get messed + * up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext filecxt) { BufFile *file = *fileptr; if (file == NULL) { + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(filecxt); + /* First write to this batch file, so open it. */ file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index 8ee59d2c71..74867c3e40 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -25,10 +25,14 @@ * * Each active hashjoin has a HashJoinTable control block, which is * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. + * for the hashjoin is kept in private memory contexts, three for each + * hashjoin: + * - HashTableContext (hashCxt): the control block associated to the hash table + * - HashBatchContext (batchCxt): storages for batches + * - HashBatchFiles (fileCxt): storage for temp files buffers + * * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * anymore. * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is @@ -39,6 +43,10 @@ * "hashCxt", while storage that is only wanted for the current batch is * allocated in the "batchCxt". By resetting the batchCxt at the end of * each batch, we free all the per-batch storage reliably and without tedium. + * Note that data associated with the temp files lives in the "fileCxt" context + * which lives during the entire join as temp files might need to survives + * batches. These files are explicitly destroyed by calling BufFileClose() + * when the code is done with them. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -350,6 +358,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext fileCxt; /* context for the BufFile related storage */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..a8f9ae1989 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext filecxt); #endif /* NODEHASHJOIN_H */ -- 2.39.2 --MP_/xtmrMo8rz+yGL.lLMmD=jly Content-Type: text/x-patch Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename=0003-v4-Add-some-debug-and-metrics.patch ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 39 +++++++++++++++++++++++++++-- src/backend/executor/nodeHashjoin.c | 14 ++++++++--- src/include/executor/hashjoin.h | 1 + 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 748c9b0024..3f1ae9755e 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or fileCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->fileCxt = AllocSetContextCreate(CurrentMemoryContext, + "HashBatchFiles", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,15 +574,21 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); + + MemoryContextSwitchTo(oldctx); } MemoryContextSwitchTo(oldcxt); @@ -934,7 +944,7 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); if (hashtable->innerBatchFile == NULL) { @@ -1020,12 +1030,19 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) } else { + MemoryContext oldctx; + /* dump it out */ Assert(batchno > curbatch); + + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, &hashtable->innerBatchFile[batchno]); + MemoryContextSwitchTo(oldctx); + hashtable->spaceUsed -= hashTupleSize; nfreed++; } @@ -1042,6 +1059,9 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) oldchunks = nextchunk; } + if (hashtable->fileCxt->mem_allocated > hashtable->spaceAllowed) + elog(WARNING, "Growing number of hash batch is exhausting memory"); + #ifdef HJDEBUG printf("Hashjoin %p: freed %ld of %ld tuples, space now %zu\n", hashtable, nfreed, ninmemory, hashtable->spaceUsed); @@ -1677,13 +1697,20 @@ ExecHashTableInsert(HashJoinTable hashtable, } else { + MemoryContext oldctx; + /* * put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + ExecHashJoinSaveTuple(tuple, hashvalue, &hashtable->innerBatchFile[batchno]); + + MemoryContextSwitchTo(oldctx); } if (shouldFree) @@ -2532,10 +2559,18 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) } else { + MemoryContext oldctx; + /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + ExecHashJoinSaveTuple(tuple, hashvalue, &hashtable->innerBatchFile[batchno]); + + MemoryContextSwitchTo(oldctx); + pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index f189fb4d28..ba1f27c2c4 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -422,6 +422,7 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) if (batchno != hashtable->curbatch && node->hj_CurSkewBucketNo == INVALID_SKEW_BUCKET_NO) { + MemoryContext oldctx; bool shouldFree; MinimalTuple mintuple = ExecFetchSlotMinimalTuple(outerTupleSlot, &shouldFree); @@ -432,9 +433,14 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) */ Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); + + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + ExecHashJoinSaveTuple(mintuple, hashvalue, &hashtable->outerBatchFile[batchno]); + MemoryContextSwitchTo(oldctx); + if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1234,9 +1240,9 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * Note: it is important always to call this in the HashBatchFiles context, + * not in a shorter-lived context; else the temp file buffers will get messed + * up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, @@ -1246,6 +1252,8 @@ ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, if (file == NULL) { + Assert(strcmp(CurrentMemoryContext->name, "HashBatchFiles") == 0); + /* First write to this batch file, so open it. */ file = BufFileCreateTemp(false); *fileptr = file; diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index acb7592ca0..d36beb7229 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -348,6 +348,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext fileCxt; /* context for the BufFile related storage */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ -- 2.39.2 --MP_/GVa.HActNIsT4j_JhNzVDcp-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH] Allocate hash batches related BufFile in a dedicated context @ 2023-03-27 13:54 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-03-27 13:54 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 43 +++++++++++++++++++++++++---- src/backend/executor/nodeHashjoin.c | 18 ++++++++---- src/include/executor/hashjoin.h | 15 ++++++++-- src/include/executor/nodeHashjoin.h | 2 +- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 748c9b0024..3da83ac22a 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or fileCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->fileCxt = AllocSetContextCreate(CurrentMemoryContext, + "HashBatchFiles", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,15 +574,21 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->fileCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); + + MemoryContextSwitchTo(oldctx); } MemoryContextSwitchTo(oldcxt); @@ -929,12 +939,18 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) nbatch = oldnbatch * 2; Assert(nbatch > 1); + elog(WARNING, "increasing number of batches from %d to %d", oldnbatch, nbatch); + + elog(LOG, "ExecHashIncreaseNumBatches ======= context stats start ======="); + MemoryContextStats(TopMemoryContext); + + #ifdef HJDEBUG printf("Hashjoin %p: increasing nbatch to %d because space = %zu\n", hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + oldcxt = MemoryContextSwitchTo(hashtable->fileCxt); if (hashtable->innerBatchFile == NULL) { @@ -1022,9 +1038,11 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) { /* dump it out */ Assert(batchno > curbatch); + ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1042,6 +1060,13 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) oldchunks = nextchunk; } + if (hashtable->fileCxt->mem_allocated > hashtable->spaceAllowed) + elog(NOTICE, + "Growing number of hash batch to %d is exhausting allowed memory (%ld > %ld)", + nbatch, + hashtable->fileCxt->mem_allocated, + hashtable->spaceAllowed); + #ifdef HJDEBUG printf("Hashjoin %p: freed %ld of %ld tuples, space now %zu\n", hashtable, nfreed, ninmemory, hashtable->spaceUsed); @@ -1063,6 +1088,9 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable); #endif } + + elog(LOG, "ExecHashIncreaseNumBatches ======= context stats end ======="); + MemoryContextStats(TopMemoryContext); } /* @@ -1681,9 +1709,11 @@ ExecHashTableInsert(HashJoinTable hashtable, * put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); } if (shouldFree) @@ -2534,8 +2564,11 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) { /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->fileCxt); + pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index f189fb4d28..6055abde49 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -432,8 +432,10 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) */ Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); + ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->fileCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1234,21 +1236,27 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * Note: it is important always to call this in the HashBatchFiles context, + * not in a shorter-lived context; else the temp file buffers will get messed + * up. */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext filecxt) { BufFile *file = *fileptr; if (file == NULL) { + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(filecxt); + /* First write to this batch file, so open it. */ file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index acb7592ca0..d759235d7f 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -25,10 +25,14 @@ * * Each active hashjoin has a HashJoinTable control block, which is * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. + * for the hashjoin is kept in private memory contexts, three for each + * hashjoin: + * - HashTableContext (hashCxt): the control block associated to the hash table + * - HashBatchContext (batchCxt): storages for batches + * - HashBatchFiles (fileCxt): storage for temp files buffers + * * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * anymore. * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is @@ -39,6 +43,10 @@ * "hashCxt", while storage that is only wanted for the current batch is * allocated in the "batchCxt". By resetting the batchCxt at the end of * each batch, we free all the per-batch storage reliably and without tedium. + * Note that data associated with the temp files lives in the "fileCxt" context + * which lives during the entire join as temp files might need to survives + * batches. These files are explicitly destroyed by calling BufFileClose() + * when the code is done with them. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -348,6 +356,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext fileCxt; /* context for the BufFile related storage */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..a8f9ae1989 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext filecxt); #endif /* NODEHASHJOIN_H */ -- 2.39.2 --MP_/cg+MWpd=CPm0_9SBk5uZwGx-- ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v8 2/3] Allocate hash batches related BufFile in a dedicated context @ 2023-05-16 13:42 Jehan-Guillaume de Rorthais <[email protected]> 0 siblings, 0 replies; 9+ messages in thread From: Jehan-Guillaume de Rorthais @ 2023-05-16 13:42 UTC (permalink / raw) --- src/backend/executor/nodeHash.c | 43 ++++++++++++++++------- src/backend/executor/nodeHashjoin.c | 22 ++++++++---- src/backend/utils/sort/sharedtuplestore.c | 8 +++++ src/include/executor/hashjoin.h | 29 ++++++++++----- src/include/executor/nodeHashjoin.h | 2 +- 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/src/backend/executor/nodeHash.c b/src/backend/executor/nodeHash.c index 5fd1c5553b..e529f50a3e 100644 --- a/src/backend/executor/nodeHash.c +++ b/src/backend/executor/nodeHash.c @@ -484,7 +484,7 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, * * The hashtable control block is just palloc'd from the executor's * per-query memory context. Everything else should be kept inside the - * subsidiary hashCxt or batchCxt. + * subsidiary hashCxt, batchCxt or spillCxt. */ hashtable = palloc_object(HashJoinTableData); hashtable->nbuckets = nbuckets; @@ -538,6 +538,10 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, "HashBatchContext", ALLOCSET_DEFAULT_SIZES); + hashtable->spillCxt = AllocSetContextCreate(hashtable->hashCxt, + "HashSpillContext", + ALLOCSET_DEFAULT_SIZES); + /* Allocate data that will live for the life of the hashjoin */ oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); @@ -570,12 +574,19 @@ ExecHashTableCreate(HashState *state, List *hashOperators, List *hashCollations, if (nbatch > 1 && hashtable->parallel_state == NULL) { + MemoryContext oldctx; + /* * allocate and initialize the file arrays in hashCxt (not needed for * parallel case which uses shared tuplestores instead of raw files) */ + oldctx = MemoryContextSwitchTo(hashtable->spillCxt); + hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldctx); + /* The files will not be opened until needed... */ /* ... but make sure we have temp tablespaces established for them */ PrepareTempTablespaces(); @@ -913,7 +924,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) int oldnbatch = hashtable->nbatch; int curbatch = hashtable->curbatch; int nbatch; - MemoryContext oldcxt; long ninmemory; long nfreed; HashMemoryChunk oldchunks; @@ -934,13 +944,16 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable, nbatch, hashtable->spaceUsed); #endif - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); - if (hashtable->innerBatchFile == NULL) { + MemoryContext oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); + /* we had no file arrays before */ hashtable->innerBatchFile = palloc0_array(BufFile *, nbatch); hashtable->outerBatchFile = palloc0_array(BufFile *, nbatch); + + MemoryContextSwitchTo(oldcxt); + /* time to establish the temp tablespaces, too */ PrepareTempTablespaces(); } @@ -951,8 +964,6 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) hashtable->outerBatchFile = repalloc0_array(hashtable->outerBatchFile, BufFile *, oldnbatch, nbatch); } - MemoryContextSwitchTo(oldcxt); - hashtable->nbatch = nbatch; /* @@ -1024,7 +1035,8 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable) Assert(batchno > curbatch); ExecHashJoinSaveTuple(HJTUPLE_MINTUPLE(hashTuple), hashTuple->hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); hashtable->spaceUsed -= hashTupleSize; nfreed++; @@ -1683,7 +1695,8 @@ ExecHashTableInsert(HashJoinTable hashtable, Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); } if (shouldFree) @@ -2664,7 +2677,8 @@ ExecHashRemoveNextSkewBucket(HashJoinTable hashtable) /* Put the tuple into a temp file for later batches */ Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(tuple, hashvalue, - &hashtable->innerBatchFile[batchno]); + &hashtable->innerBatchFile[batchno], + hashtable->spillCxt); pfree(hashTuple); hashtable->spaceUsed -= tupleSize; hashtable->spaceUsedSkew -= tupleSize; @@ -3093,8 +3107,11 @@ ExecParallelHashJoinSetUpBatches(HashJoinTable hashtable, int nbatch) pstate->nbatch = nbatch; batches = dsa_get_address(hashtable->area, pstate->batches); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* + * Use hash join spill memory context to allocate accessors and their + * buffers + */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = nbatch; @@ -3196,8 +3213,8 @@ ExecParallelHashEnsureBatchAccessors(HashJoinTable hashtable) */ Assert(DsaPointerIsValid(pstate->batches)); - /* Use hash join memory context. */ - oldcxt = MemoryContextSwitchTo(hashtable->hashCxt); + /* Use hash join spill memory context to allocate accessors */ + oldcxt = MemoryContextSwitchTo(hashtable->spillCxt); /* Allocate this backend's accessor array. */ hashtable->nbatch = pstate->nbatch; diff --git a/src/backend/executor/nodeHashjoin.c b/src/backend/executor/nodeHashjoin.c index 93a78f6f74..e9e58e9647 100644 --- a/src/backend/executor/nodeHashjoin.c +++ b/src/backend/executor/nodeHashjoin.c @@ -491,7 +491,8 @@ ExecHashJoinImpl(PlanState *pstate, bool parallel) Assert(parallel_state == NULL); Assert(batchno > hashtable->curbatch); ExecHashJoinSaveTuple(mintuple, hashvalue, - &hashtable->outerBatchFile[batchno]); + &hashtable->outerBatchFile[batchno], + hashtable->spillCxt); if (shouldFree) heap_free_minimal_tuple(mintuple); @@ -1313,21 +1314,30 @@ ExecParallelHashJoinNewBatch(HashJoinState *hjstate) * The data recorded in the file for each tuple is its hash value, * then the tuple in MinimalTuple format. * - * Note: it is important always to call this in the regular executor - * context, not in a shorter-lived context; else the temp file buffers - * will get messed up. + * If this is the first write to the batch file, this function first + * create it. The associated BufFile buffer is allocated in the given + * context. It is important to always give the HashSpillContext + * context. First to avoid a shorter-lived context, else the temp file + * buffers will get messed up. Second, for a better accounting of the + * spilling memory consumption. + * */ void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr) + BufFile **fileptr, MemoryContext cxt) { BufFile *file = *fileptr; if (file == NULL) { - /* First write to this batch file, so open it. */ + MemoryContext oldctx; + + oldctx = MemoryContextSwitchTo(cxt); + file = BufFileCreateTemp(false); *fileptr = file; + + MemoryContextSwitchTo(oldctx); } BufFileWrite(file, &hashvalue, sizeof(uint32)); diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c index 0831249159..236be65f22 100644 --- a/src/backend/utils/sort/sharedtuplestore.c +++ b/src/backend/utils/sort/sharedtuplestore.c @@ -308,11 +308,15 @@ sts_puttuple(SharedTuplestoreAccessor *accessor, void *meta_data, { SharedTuplestoreParticipant *participant; char name[MAXPGPATH]; + MemoryContext oldcxt; /* Create one. Only this backend will write into it. */ sts_filename(name, accessor, accessor->participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->write_file = BufFileCreateFileSet(&accessor->fileset->fs, name); + MemoryContextSwitchTo(oldcxt); /* Set up the shared state for this backend's file. */ participant = &accessor->sts->participants[accessor->participant]; @@ -527,11 +531,15 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data) if (accessor->read_file == NULL) { char name[MAXPGPATH]; + MemoryContext oldcxt; sts_filename(name, accessor, accessor->read_participant); + + oldcxt = MemoryContextSwitchTo(accessor->context); accessor->read_file = BufFileOpenFileSet(&accessor->fileset->fs, name, O_RDONLY, false); + MemoryContextSwitchTo(oldcxt); } /* Seek and load the chunk header. */ diff --git a/src/include/executor/hashjoin.h b/src/include/executor/hashjoin.h index 8ee59d2c71..ac27222d18 100644 --- a/src/include/executor/hashjoin.h +++ b/src/include/executor/hashjoin.h @@ -23,12 +23,12 @@ /* ---------------------------------------------------------------- * hash-join hash table structures * - * Each active hashjoin has a HashJoinTable control block, which is - * palloc'd in the executor's per-query context. All other storage needed - * for the hashjoin is kept in private memory contexts, two for each hashjoin. - * This makes it easy and fast to release the storage when we don't need it - * anymore. (Exception: data associated with the temp files lives in the - * per-query context too, since we always call buffile.c in that context.) + * Each active hashjoin has a HashJoinTable structure, which is + * palloc'd in the executor's per-query context. Other storages needed for + * each hashjoin is kept in child contexts, three for each hashjoin: + * - HashTableContext (hashCxt): the parent hash table storage context + * - HashSpillContext (spillCxt): storage for temp files buffers + * - HashBatchContext (batchCxt): storage for a batch in serial hash join * * The hashtable contexts are made children of the per-query context, ensuring * that they will be discarded at end of statement even if the join is @@ -36,9 +36,19 @@ * be cleaned up by the virtual file manager in event of an error.) * * Storage that should live through the entire join is allocated from the - * "hashCxt", while storage that is only wanted for the current batch is - * allocated in the "batchCxt". By resetting the batchCxt at the end of - * each batch, we free all the per-batch storage reliably and without tedium. + * "hashCxt" (mainly hash's meta datas). Also, the "hashCxt" context is the + * parent of "spillCxt" and "batchCxt". It makes it easy and fast to release + * the storage when we don't need it anymore. + * + * Data associated with temp files lives in the "spillCxt" context which lives + * during the entire join as temp files might need to survives batches. These + * files are explicitly destroyed by calling BufFileClose() when the code is + * done with them. The aim of this context is to help accounting the memory + * allocations dedicated to temp files and their buffers. + * + * Finaly, storage that is only wanted for the current batch is allocated in + * the "batchCxt". By resetting the batchCxt at the end of each batch, we free + * all the per-batch storage reliably and without tedium. * * During first scan of inner relation, we get its tuples from executor. * If nbatch > 1 then tuples that don't belong in first batch get saved @@ -350,6 +360,7 @@ typedef struct HashJoinTableData MemoryContext hashCxt; /* context for whole-hash-join storage */ MemoryContext batchCxt; /* context for this-batch-only storage */ + MemoryContext spillCxt; /* context for spilling to temp files */ /* used for dense allocation of tuples (into linked chunks) */ HashMemoryChunk chunks; /* one list for the whole batch */ diff --git a/src/include/executor/nodeHashjoin.h b/src/include/executor/nodeHashjoin.h index d367070883..fb33994366 100644 --- a/src/include/executor/nodeHashjoin.h +++ b/src/include/executor/nodeHashjoin.h @@ -29,6 +29,6 @@ extern void ExecHashJoinInitializeWorker(HashJoinState *state, ParallelWorkerContext *pwcxt); extern void ExecHashJoinSaveTuple(MinimalTuple tuple, uint32 hashvalue, - BufFile **fileptr); + BufFile **fileptr, MemoryContext cxt); #endif /* NODEHASHJOIN_H */ -- 2.40.1 --MP_/KUKoyOpWagGRmSskvJu5/tn-- ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2023-05-16 13:42 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2023-03-27 13:54 [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH 2/3] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH 2/4] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-03-27 13:54 [PATCH] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]> 2023-05-16 13:42 [PATCH v8 2/3] Allocate hash batches related BufFile in a dedicated context Jehan-Guillaume de Rorthais <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox