public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v12 1/5] Allow CREATE INDEX CONCURRENTLY on partitioned table 80+ messages / 3 participants [nested] [flat]
* [PATCH v12 1/5] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 142 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 173 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f9f3ff3b62..c513e8a6bd 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,18 +1178,29 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1243,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1319,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1378,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,51 +1399,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1617,6 +1628,57 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { .options = REINDEXOPT_CONCURRENTLY }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --MfFXiAuoTsnnDAfZ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0002-Add-SKIPVALID-flag-for-more-integration.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH 1/5] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 965dcf472c..7c75119d78 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 8bc652ecd3..9ab1a66971 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1626,6 +1637,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --dTy3Mrz/UPE2dbVg Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) https://www.postgresql.org/message-id/flat/[email protected] --- doc/src/sgml/ddl.sgml | 4 +- doc/src/sgml/ref/create_index.sgml | 14 +- src/backend/commands/indexcmds.c | 201 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 127 +++++++++++++++- src/test/regress/sql/indexing.sql | 26 +++- 5 files changed, 297 insertions(+), 75 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 58aaa691c6a..afa982154a8 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4161,9 +4161,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 so that they are applied automatically to the entire hierarchy. This is very convenient, as not only will the existing partitions become indexed, but - also any partitions that are created in the future will. One limitation is - that it's not possible to use the <literal>CONCURRENTLY</literal> - qualifier when creating such a partitioned index. To avoid long lock + also any partitions that are created in the future will. To avoid long lock times, it is possible to use <command>CREATE INDEX ON ONLY</command> the partitioned table; such an index is marked invalid, and the partitions do not get the index applied automatically. The indexes on partitions can diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 40986aa502f..b05102efdaf 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -645,7 +645,10 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class= <para> If a problem arises while scanning the table, such as a deadlock or a uniqueness violation in a unique index, the <command>CREATE INDEX</command> - command will fail but leave behind an <quote>invalid</quote> index. This index + command will fail but leave behind an <quote>invalid</quote> index. + If this happens while build an index concurrently on a partitioned + table, the command can also leave behind <quote>valid</quote> or + <quote>invalid</quote> indexes on table partitions. The invalid index will be ignored for querying purposes because it might be incomplete; however it will still consume update overhead. The <application>psql</application> <command>\d</command> command will report such an index as <literal>INVALID</literal>: @@ -692,15 +695,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index baf3e6e57a5..dfe64052b81 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -92,6 +92,11 @@ static char *ChooseIndexName(const char *tabname, Oid namespaceId, bool primary, bool isconstraint); static char *ChooseIndexNameAddition(List *colnames); static List *ChooseIndexColumnNames(List *indexElems); +static void DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, + IndexInfo *indexInfo, + LOCKTAG heaplocktag, + LockRelId heaprelid); static void ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, @@ -555,7 +560,6 @@ DefineIndex(Oid relationId, bool amissummarizing; amoptions_function amoptions; bool partitioned; - bool safe_index; Datum reloptions; int16 *coloptions; IndexInfo *indexInfo; @@ -563,12 +567,10 @@ DefineIndex(Oid relationId, bits16 constr_flags; int numberOfAttributes; int numberOfKeyAttributes; - TransactionId limitXmin; ObjectAddress address; LockRelId heaprelid; LOCKTAG heaplocktag; LOCKMODE lockmode; - Snapshot snapshot; Oid root_save_userid; int root_save_sec_context; int root_save_nestlevel; @@ -699,20 +701,6 @@ DefineIndex(Oid relationId, * partition. */ partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; - if (partitioned) - { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); - } /* * Don't try to CREATE INDEX on temp tables of other backends. @@ -1102,10 +1090,6 @@ DefineIndex(Oid relationId, } } - /* Is index safe for others to ignore? See set_indexsafe_procflags() */ - safe_index = indexInfo->ii_Expressions == NIL && - indexInfo->ii_Predicate == NIL; - /* * Report index creation if appropriate (delay this till after most of the * error checks) @@ -1170,6 +1154,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1519,21 +1508,7 @@ DefineIndex(Oid relationId, */ if (invalidate_parent) { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); /* * CCI here to make this update visible, in case this recurses @@ -1545,37 +1520,49 @@ DefineIndex(Oid relationId, /* * Indexes on partitioned tables are not themselves built, so we're - * done here. + * done here in the non-concurrent case. */ - AtEOXact_GUC(false, root_save_nestlevel); - SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - table_close(rel, NoLock); - if (!OidIsValid(parentIndexId)) - pgstat_progress_end_command(); - else + if (!concurrent) { - /* Update progress for an intermediate partitioned index itself */ - pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); - } + AtEOXact_GUC(false, root_save_nestlevel); + SetUserIdAndSecContext(root_save_userid, root_save_sec_context); + table_close(rel, NoLock); - return address; + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); + else + { + /* + * Update progress for an intermediate partitioned index + * itself + */ + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + } + + return address; + } } AtEOXact_GUC(false, root_save_nestlevel); SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - if (!concurrent) + /* + * All done in the non-concurrent case, and when building catalog entries + * of partitions for CIC. + */ + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ table_close(rel, NoLock); /* * If this is the top-level index, the command is done overall; - * otherwise, increment progress to report one child index is done. + * otherwise (when being called recursively), increment progress to + * report that one child index is done. Except in the concurrent + * (catalog-only) case, which is handled later. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); - else + else if (!concurrent) pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); return address; @@ -1586,6 +1573,114 @@ DefineIndex(Oid relationId, SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); table_close(rel, NoLock); + if (!partitioned) + { + /* CREATE INDEX CONCURRENTLY on a nonpartitioned table */ + DefineIndexConcurrentInternal(relationId, indexRelationId, + indexInfo, heaplocktag, heaprelid); + pgstat_progress_end_command(); + return address; + } + else + { + /* + * For CIC on a partitioned table, finish by building indexes on + * partitions + */ + + ListCell *lc; + List *childs; + List *tosetvalid = NIL; + MemoryContext cic_context, + old_context; + + /* Create special memory context for cross-transaction storage */ + cic_context = AllocSetContextCreate(PortalContext, + "Create index concurrently", + ALLOCSET_DEFAULT_SIZES); + + old_context = MemoryContextSwitchTo(cic_context); + childs = find_all_inheritors(indexRelationId, ShareLock, NULL); + MemoryContextSwitchTo(old_context); + + foreach(lc, childs) + { + Oid indrelid = lfirst_oid(lc); + Oid tabrelid; + char relkind; + + /* + * Pre-existing partitions which were ATTACHED were already + * counted in the progress report. + */ + if (get_index_isvalid(indrelid)) + continue; + + /* + * Partitioned indexes are counted in the progress report, but + * don't need to be further processed. + */ + relkind = get_rel_relkind(indrelid); + if (!RELKIND_HAS_STORAGE(relkind)) + { + /* The toplevel index doesn't count towards "partitions done" */ + if (indrelid != indexRelationId) + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + + /* + * Build up a list of all the intermediate partitioned tables + * which will later need to be set valid. + */ + old_context = MemoryContextSwitchTo(cic_context); + tosetvalid = lappend_oid(tosetvalid, indrelid); + MemoryContextSwitchTo(old_context); + continue; + } + + rel = table_open(relationId, ShareUpdateExclusiveLock); + heaprelid = rel->rd_lockInfo.lockRelId; + table_close(rel, ShareUpdateExclusiveLock); + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + + /* Process each partition in a separate transaction */ + tabrelid = IndexGetRelation(indrelid, false); + DefineIndexConcurrentInternal(tabrelid, indrelid, indexInfo, + heaplocktag, heaprelid); + + PushActiveSnapshot(GetTransactionSnapshot()); + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + } + + /* Set as valid all partitioned indexes, including the parent */ + foreach(lc, tosetvalid) + { + Oid indrelid = lfirst_oid(lc); + + index_set_state_flags(indrelid, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indrelid, INDEX_CREATE_SET_VALID); + } + + MemoryContextDelete(cic_context); + pgstat_progress_end_command(); + PopActiveSnapshot(); + return address; + } +} + + +static void +DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, IndexInfo *indexInfo, + LOCKTAG heaplocktag, LockRelId heaprelid) +{ + TransactionId limitXmin; + Snapshot snapshot; + + /* Is index safe for others to ignore? See set_indexsafe_procflags() */ + bool safe_index = indexInfo->ii_Expressions == NIL && + indexInfo->ii_Predicate == NIL; + /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That @@ -1781,10 +1876,6 @@ DefineIndex(Oid relationId, * Last thing to do is release the session-level lock on the parent table. */ UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock); - - pgstat_progress_end_command(); - - return address; } diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 2be8ffa7ec4..aefa203b14f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,130 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx1" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 3 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart11 + Partitioned table "public.idxpart11" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart1 FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart11_a_idx" btree (a) + "idxpart11_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart111 + Partitioned table "public.idxpart111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart11 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart111_a_idx" btree (a) + "idxpart111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart1111 + Partitioned table "public.idxpart1111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart111 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart1111_a_idx" btree (a) + "idxpart1111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 0 + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" UNIQUE, btree (a) INVALID + +\d idxpart3 + Partitioned table "public.idxpart3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (30) TO (40) +Partition key: RANGE (a) +Indexes: + "idxpart3_a_idx" btree (a) + "idxpart3_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart31 + Table "public.idxpart31" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart3 DEFAULT +Indexes: + "idxpart31_a_idx" btree (a) + "idxpart31_a_idx1" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index b69c41832ca..5ddeaf1c613 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,30 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; + +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart11 +\d idxpart111 +\d idxpart1111 +\d idxpart2 +\d idxpart3 +\d idxpart31 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.34.1 --9G1BO/DxFoGcNMgy-- ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) --- doc/src/sgml/ddl.sgml | 4 +- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 172 +++++++++++++++++-------- src/test/regress/expected/indexing.out | 127 +++++++++++++++++- src/test/regress/sql/indexing.sql | 26 +++- 5 files changed, 268 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 38618de01c5..cd72b455447 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4163,9 +4163,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 so that they are applied automatically to the entire hierarchy. This is very convenient, as not only will the existing partitions become indexed, but - also any partitions that are created in the future will. One limitation is - that it's not possible to use the <literal>CONCURRENTLY</literal> - qualifier when creating such a partitioned index. To avoid long lock + also any partitions that are created in the future will. To avoid long lock times, it is possible to use <command>CREATE INDEX ON ONLY</command> the partitioned table; such an index is marked invalid, and the partitions do not get the index applied automatically. The indexes on partitions can diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 40986aa502f..fc8cda655f0 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -692,15 +692,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index b5b860c3abf..cfab45b9992 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -92,6 +92,11 @@ static char *ChooseIndexName(const char *tabname, Oid namespaceId, bool primary, bool isconstraint); static char *ChooseIndexNameAddition(List *colnames); static List *ChooseIndexColumnNames(List *indexElems); +static void DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, + IndexInfo *indexInfo, + LOCKTAG heaplocktag, + LockRelId heaprelid); static void ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, @@ -551,7 +556,6 @@ DefineIndex(Oid relationId, bool amcanorder; amoptions_function amoptions; bool partitioned; - bool safe_index; Datum reloptions; int16 *coloptions; IndexInfo *indexInfo; @@ -559,12 +563,10 @@ DefineIndex(Oid relationId, bits16 constr_flags; int numberOfAttributes; int numberOfKeyAttributes; - TransactionId limitXmin; ObjectAddress address; LockRelId heaprelid; LOCKTAG heaplocktag; LOCKMODE lockmode; - Snapshot snapshot; Oid root_save_userid; int root_save_sec_context; int root_save_nestlevel; @@ -697,17 +699,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1079,10 +1070,6 @@ DefineIndex(Oid relationId, } } - /* Is index safe for others to ignore? See set_indexsafe_procflags() */ - safe_index = indexInfo->ii_Expressions == NIL && - indexInfo->ii_Predicate == NIL; - /* * Report index creation if appropriate (delay this till after most of the * error checks) @@ -1147,6 +1134,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1427,12 +1419,15 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + SetUserIdAndSecContext(child_save_userid, child_save_sec_context); } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1444,46 +1439,39 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're - * done here. + * done here in the non-concurrent case. */ - AtEOXact_GUC(false, root_save_nestlevel); - SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - table_close(rel, NoLock); - if (!OidIsValid(parentIndexId)) - pgstat_progress_end_command(); - return address; + + if (!concurrent) + { + AtEOXact_GUC(false, root_save_nestlevel); + SetUserIdAndSecContext(root_save_userid, root_save_sec_context); + table_close(rel, NoLock); + + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); + + return address; + } } AtEOXact_GUC(false, root_save_nestlevel); SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - if (!concurrent) + /* + * All done in the non-concurrent case, and when building catalog entries + * of partitions for CIC. + */ + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ + /* If this is the top-level index, the command is complete. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1495,6 +1483,92 @@ DefineIndex(Oid relationId, SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); table_close(rel, NoLock); + if (!partitioned) + { + DefineIndexConcurrentInternal(relationId, indexRelationId, + indexInfo, heaplocktag, heaprelid); + pgstat_progress_end_command(); + return address; + } + else + { + /* finish CIC by building indexes on partitions */ + ListCell *lc; + List *childs; + int npart = 0; + MemoryContext cic_context, + old_context; + + /* + * Create special memory context for cross-transaction storage. + */ + cic_context = AllocSetContextCreate(PortalContext, + "Create index concurrently", + ALLOCSET_DEFAULT_SIZES); + + old_context = MemoryContextSwitchTo(cic_context); + childs = find_all_inheritors(indexRelationId, ShareLock, NULL); + MemoryContextSwitchTo(old_context); + + foreach(lc, childs) + { + Oid indrelid = lfirst_oid(lc); + Oid tabrelid = IndexGetRelation(indrelid, false); + + if (RELKIND_HAS_STORAGE(get_rel_relkind(indrelid)) && + !get_index_isvalid(indrelid)) + { + rel = table_open(relationId, ShareUpdateExclusiveLock); + heaprelid = rel->rd_lockInfo.lockRelId; + table_close(rel, ShareUpdateExclusiveLock); + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + + /* Process each partition in a separate transaction */ + DefineIndexConcurrentInternal(tabrelid, indrelid, indexInfo, + heaplocktag, heaprelid); + + PushActiveSnapshot(GetTransactionSnapshot()); + } + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + ++npart); + } + + /* Set all indexes as valid, including the parent */ + foreach(lc, childs) + { + Oid indrelid = lfirst_oid(lc); + + if (get_rel_relkind(indrelid) != RELKIND_PARTITIONED_INDEX) + continue; + if (get_index_isvalid(indrelid)) + continue; + + index_set_state_flags(indrelid, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indrelid, INDEX_CREATE_SET_VALID); + } + + MemoryContextDelete(cic_context); + pgstat_progress_end_command(); + PopActiveSnapshot(); + return address; + } +} + + +static void +DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, IndexInfo *indexInfo, + LOCKTAG heaplocktag, LockRelId heaprelid) +{ + TransactionId limitXmin; + Snapshot snapshot; + + /* Is index safe for others to ignore? See set_indexsafe_procflags() */ + bool safe_index = indexInfo->ii_Expressions == NIL && + indexInfo->ii_Predicate == NIL; + /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That @@ -1690,10 +1764,6 @@ DefineIndex(Oid relationId, * Last thing to do is release the session-level lock on the parent table. */ UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock); - - pgstat_progress_end_command(); - - return address; } diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 1bdd430f063..f1beee6d240 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,130 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx1" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 3 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart11 + Partitioned table "public.idxpart11" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart1 FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart11_a_idx" btree (a) + "idxpart11_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart111 + Partitioned table "public.idxpart111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart11 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart111_a_idx" btree (a) + "idxpart111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart1111 + Partitioned table "public.idxpart1111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart111 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart1111_a_idx" btree (a) + "idxpart1111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 0 + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" UNIQUE, btree (a) INVALID + +\d idxpart3 + Partitioned table "public.idxpart3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (30) TO (40) +Partition key: RANGE (a) +Indexes: + "idxpart3_a_idx" btree (a) + "idxpart3_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart31 + Table "public.idxpart31" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart3 DEFAULT +Indexes: + "idxpart31_a_idx" btree (a) + "idxpart31_a_idx1" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 429120e7104..fb0baedcc28 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,30 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; + +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart11 +\d idxpart111 +\d idxpart1111 +\d idxpart2 +\d idxpart3 +\d idxpart31 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.25.1 --zCKi3GIZzVBPywwA-- ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) https://www.postgresql.org/message-id/flat/[email protected] --- doc/src/sgml/ddl.sgml | 4 +- doc/src/sgml/ref/create_index.sgml | 14 +- src/backend/commands/indexcmds.c | 200 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 127 +++++++++++++++- src/test/regress/sql/indexing.sql | 26 +++- 5 files changed, 297 insertions(+), 74 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 91c036d1cbe..64efdf1e879 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4178,9 +4178,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 so that they are applied automatically to the entire hierarchy. This is very convenient, as not only will the existing partitions become indexed, but - also any partitions that are created in the future will. One limitation is - that it's not possible to use the <literal>CONCURRENTLY</literal> - qualifier when creating such a partitioned index. To avoid long lock + also any partitions that are created in the future will. To avoid long lock times, it is possible to use <command>CREATE INDEX ON ONLY</command> the partitioned table; such an index is marked invalid, and the partitions do not get the index applied automatically. The indexes on partitions can diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 40986aa502f..b05102efdaf 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -645,7 +645,10 @@ CREATE [ UNIQUE ] INDEX [ CONCURRENTLY ] [ [ IF NOT EXISTS ] <replaceable class= <para> If a problem arises while scanning the table, such as a deadlock or a uniqueness violation in a unique index, the <command>CREATE INDEX</command> - command will fail but leave behind an <quote>invalid</quote> index. This index + command will fail but leave behind an <quote>invalid</quote> index. + If this happens while build an index concurrently on a partitioned + table, the command can also leave behind <quote>valid</quote> or + <quote>invalid</quote> indexes on table partitions. The invalid index will be ignored for querying purposes because it might be incomplete; however it will still consume update overhead. The <application>psql</application> <command>\d</command> command will report such an index as <literal>INVALID</literal>: @@ -692,15 +695,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 3ec8b5cca6c..daba8b67dbe 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -93,6 +93,11 @@ static char *ChooseIndexName(const char *tabname, Oid namespaceId, bool primary, bool isconstraint); static char *ChooseIndexNameAddition(List *colnames); static List *ChooseIndexColumnNames(List *indexElems); +static void DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, + IndexInfo *indexInfo, + LOCKTAG heaplocktag, + LockRelId heaprelid); static void ReindexIndex(RangeVar *indexRelation, ReindexParams *params, bool isTopLevel); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, @@ -559,7 +564,6 @@ DefineIndex(Oid relationId, bool amissummarizing; amoptions_function amoptions; bool partitioned; - bool safe_index; Datum reloptions; int16 *coloptions; IndexInfo *indexInfo; @@ -567,12 +571,10 @@ DefineIndex(Oid relationId, bits16 constr_flags; int numberOfAttributes; int numberOfKeyAttributes; - TransactionId limitXmin; ObjectAddress address; LockRelId heaprelid; LOCKTAG heaplocktag; LOCKMODE lockmode; - Snapshot snapshot; Oid root_save_userid; int root_save_sec_context; int root_save_nestlevel; @@ -705,17 +707,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1089,10 +1080,6 @@ DefineIndex(Oid relationId, } } - /* Is index safe for others to ignore? See set_indexsafe_procflags() */ - safe_index = indexInfo->ii_Expressions == NIL && - indexInfo->ii_Predicate == NIL; - /* * Report index creation if appropriate (delay this till after most of the * error checks) @@ -1157,6 +1144,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1494,58 +1486,54 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're - * done here. + * done here in the non-concurrent case. */ - AtEOXact_GUC(false, root_save_nestlevel); - SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - table_close(rel, NoLock); - if (!OidIsValid(parentIndexId)) - pgstat_progress_end_command(); - else + if (!concurrent) { - /* Update progress for an intermediate partitioned index itself */ - pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); - } + AtEOXact_GUC(false, root_save_nestlevel); + SetUserIdAndSecContext(root_save_userid, root_save_sec_context); + table_close(rel, NoLock); - return address; + if (!OidIsValid(parentIndexId)) + pgstat_progress_end_command(); + else + { + /* + * Update progress for an intermediate partitioned index + * itself + */ + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + } + + return address; + } } AtEOXact_GUC(false, root_save_nestlevel); SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - if (!concurrent) + /* + * All done in the non-concurrent case, and when building catalog entries + * of partitions for CIC. + */ + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ table_close(rel, NoLock); /* * If this is the top-level index, the command is done overall; - * otherwise, increment progress to report one child index is done. + * otherwise (when being called recursively), increment progress to + * report that one child index is done. Except in the concurrent + * (catalog-only) case, which is handled later. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); - else + else if (!concurrent) pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); return address; @@ -1556,6 +1544,114 @@ DefineIndex(Oid relationId, SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); table_close(rel, NoLock); + if (!partitioned) + { + /* CREATE INDEX CONCURRENTLY on a nonpartitioned table */ + DefineIndexConcurrentInternal(relationId, indexRelationId, + indexInfo, heaplocktag, heaprelid); + pgstat_progress_end_command(); + return address; + } + else + { + /* + * For CIC on a partitioned table, finish by building indexes on + * partitions + */ + + ListCell *lc; + List *childs; + List *partitioned = NIL; + MemoryContext cic_context, + old_context; + + /* Create special memory context for cross-transaction storage */ + cic_context = AllocSetContextCreate(PortalContext, + "Create index concurrently", + ALLOCSET_DEFAULT_SIZES); + + old_context = MemoryContextSwitchTo(cic_context); + childs = find_all_inheritors(indexRelationId, ShareLock, NULL); + MemoryContextSwitchTo(old_context); + + foreach(lc, childs) + { + Oid indrelid = lfirst_oid(lc); + Oid tabrelid; + char relkind; + + /* + * Pre-existing partitions which were ATTACHED were already + * counted in the progress report. + */ + if (get_index_isvalid(indrelid)) + continue; + + /* + * Partitioned indexes are counted in the progress report, but + * don't need to be further processed. + */ + relkind = get_rel_relkind(indrelid); + if (!RELKIND_HAS_STORAGE(relkind)) + { + /* The toplevel index doesn't count towards "partitions done" */ + if (indrelid != indexRelationId) + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + + /* + * Build up a list of all the intermediate partitioned tables + * which will later need to be set valid. + */ + old_context = MemoryContextSwitchTo(cic_context); + partitioned = lappend_oid(partitioned, indrelid); + MemoryContextSwitchTo(old_context); + continue; + } + + rel = table_open(relationId, ShareUpdateExclusiveLock); + heaprelid = rel->rd_lockInfo.lockRelId; + table_close(rel, ShareUpdateExclusiveLock); + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + + /* Process each partition in a separate transaction */ + tabrelid = IndexGetRelation(indrelid, false); + DefineIndexConcurrentInternal(tabrelid, indrelid, indexInfo, + heaplocktag, heaprelid); + + PushActiveSnapshot(GetTransactionSnapshot()); + pgstat_progress_incr_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, 1); + } + + /* Set as valid all partitioned indexes, including the parent */ + foreach(lc, partitioned) + { + Oid indrelid = lfirst_oid(lc); + + index_set_state_flags(indrelid, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indrelid, INDEX_CREATE_SET_VALID); + } + + MemoryContextDelete(cic_context); + pgstat_progress_end_command(); + PopActiveSnapshot(); + return address; + } +} + + +static void +DefineIndexConcurrentInternal(Oid relationId, + Oid indexRelationId, IndexInfo *indexInfo, + LOCKTAG heaplocktag, LockRelId heaprelid) +{ + TransactionId limitXmin; + Snapshot snapshot; + + /* Is index safe for others to ignore? See set_indexsafe_procflags() */ + bool safe_index = indexInfo->ii_Expressions == NIL && + indexInfo->ii_Predicate == NIL; + /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That @@ -1751,10 +1847,6 @@ DefineIndex(Oid relationId, * Last thing to do is release the session-level lock on the parent table. */ UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock); - - pgstat_progress_end_command(); - - return address; } diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 1bdd430f063..f1beee6d240 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,130 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx1" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 3 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart11 + Partitioned table "public.idxpart11" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart1 FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart11_a_idx" btree (a) + "idxpart11_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart111 + Partitioned table "public.idxpart111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart11 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart111_a_idx" btree (a) + "idxpart111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart1111 + Partitioned table "public.idxpart1111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart111 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart1111_a_idx" btree (a) + "idxpart1111_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 0 + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" UNIQUE, btree (a) INVALID + +\d idxpart3 + Partitioned table "public.idxpart3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (30) TO (40) +Partition key: RANGE (a) +Indexes: + "idxpart3_a_idx" btree (a) + "idxpart3_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart31 + Table "public.idxpart31" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart3 DEFAULT +Indexes: + "idxpart31_a_idx" btree (a) + "idxpart31_a_idx1" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 429120e7104..fb0baedcc28 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,30 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; + +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart2 (a); -- leaf +create index concurrently on idxpart (a); -- partitioned +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart11 +\d idxpart111 +\d idxpart1111 +\d idxpart2 +\d idxpart3 +\d idxpart31 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.34.1 --naDu/P2UbiQYQWIS-- ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ddl.sgml | 4 +- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 214 +++++++++++++++++++------ src/include/catalog/index.h | 1 + src/test/regress/expected/indexing.out | 136 +++++++++++++++- src/test/regress/sql/indexing.sql | 26 ++- 6 files changed, 320 insertions(+), 70 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 03c01937094..fd56e21ef49 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -4131,9 +4131,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02 so that they are applied automatically to the entire hierarchy. This is very convenient, as not only will the existing partitions become indexed, but - also any partitions that are created in the future will. One limitation is - that it's not possible to use the <literal>CONCURRENTLY</literal> - qualifier when creating such a partitioned index. To avoid long lock + also any partitions that are created in the future will. To avoid long lock times, it is possible to use <command>CREATE INDEX ON ONLY</command> the partitioned table; such an index is marked invalid, and the partitions do not get the index applied automatically. The indexes on partitions can diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 40986aa502f..fc8cda655f0 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -692,15 +692,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 91cee27743d..bb98e745267 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -71,6 +71,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -104,7 +105,9 @@ static void reindex_error_callback(void *arg); static void ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel); static void ReindexMultipleInternal(List *relids, - ReindexParams *params); + ReindexParams *params, + Oid parent, + int npart); static bool ReindexRelationConcurrently(Oid relationOid, ReindexParams *params); static void update_relispartition(Oid relationId, bool newval); @@ -697,17 +700,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1147,6 +1139,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1212,17 +1209,30 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel, true); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc_array(Oid, nparts); bool invalidate_parent = false; Relation parentIndex; TupleDesc parentDesc; + char *relname; pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); /* Make a local copy of partdesc->oids[], just for safety */ memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + relname = pstrdup(RelationGetRelationName(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); /* * We'll need an IndexInfo describing the parent index. The one @@ -1235,8 +1245,6 @@ DefineIndex(Oid relationId, parentIndex = index_open(indexRelationId, lockmode); indexInfo = BuildIndexInfo(parentIndex); - parentDesc = RelationGetDescr(rel); - /* * For each partition, scan all existing indexes; if one matches * our index definition and is not already attached to some other @@ -1276,9 +1284,9 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); AtEOXact_GUC(false, child_save_nestlevel); SetUserIdAndSecContext(child_save_userid, @@ -1364,10 +1372,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1426,14 +1438,24 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + SetUserIdAndSecContext(child_save_userid, child_save_sec_context); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } + pfree(relname); index_close(parentIndex, lockmode); @@ -1443,46 +1465,40 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ + AtEOXact_GUC(false, root_save_nestlevel); SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - table_close(rel, NoLock); + if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } AtEOXact_GUC(false, root_save_nestlevel); SetUserIdAndSecContext(root_save_userid, root_save_sec_context); - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1695,6 +1711,70 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + List *parentindexes = NIL; + int npart = 1; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY | REINDEXOPT_REPORT_CREATE_PART + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_all_inheritors(indexRelationId, ShareLock, NULL); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + char partkind = get_rel_relkind(partoid); + + if (partkind == RELKIND_PARTITIONED_INDEX) + { + oldcontext = MemoryContextSwitchTo(ind_context); + parentindexes = lappend_oid(parentindexes, partoid); + MemoryContextSwitchTo(oldcontext); + } + + // npart++ ? + if (!RELKIND_HAS_STORAGE(partkind) || get_index_isvalid(partoid)) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms, indexRelationId, npart); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though it's + * meaningless for an index without storage). This must be done only + * while holding a lock which precludes adding partitions. + * See also: validatePartitionedIndex(). + */ + foreach (lc, parentindexes) + { + Oid partoid = lfirst_oid(lc); + index_set_state_flags(partoid, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(partoid, INDEX_CREATE_SET_VALID); + } +} /* * CheckMutability @@ -3084,7 +3164,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Process each relation listed in a separate transaction. Note that this * commits and then starts a new transaction immediately. */ - ReindexMultipleInternal(relids, params); + ReindexMultipleInternal(relids, params, InvalidOid, 0); MemoryContextDelete(private_context); } @@ -3120,6 +3200,7 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) char relkind = get_rel_relkind(relid); char *relname = get_rel_name(relid); char *relnamespace = get_namespace_name(get_rel_namespace(relid)); + int npart = 1; MemoryContext reindex_context; List *inhoids; ListCell *lc; @@ -3190,7 +3271,7 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * Process each partition listed in a separate transaction. Note that * this commits and then starts a new transaction immediately. */ - ReindexMultipleInternal(partitions, params); + ReindexMultipleInternal(partitions, params, relid, npart); /* * Clean up working storage --- note we must do this after @@ -3208,7 +3289,7 @@ ReindexPartitions(Oid relid, ReindexParams *params, bool isTopLevel) * and starts a new transaction when finished. */ static void -ReindexMultipleInternal(List *relids, ReindexParams *params) +ReindexMultipleInternal(List *relids, ReindexParams *params, Oid parent, int npart) { ListCell *l; @@ -3302,6 +3383,29 @@ ReindexMultipleInternal(List *relids, ReindexParams *params) } CommitTransactionCommand(); + + if (params->options & REINDEXOPT_REPORT_CREATE_PART) + { + /* + * Update pgstat progress report to indicate that create index on + * partition was finished. + */ + const int progress_cols[] = { + PROGRESS_CREATEIDX_COMMAND, + PROGRESS_CREATEIDX_INDEX_OID, + PROGRESS_CREATEIDX_PHASE, + PROGRESS_CREATEIDX_PARTITIONS_DONE + }; + const int64 progress_vals[] = { + PROGRESS_CREATEIDX_COMMAND_CREATE_CONCURRENTLY, + parent, + PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN, + npart++ + }; + + pgstat_progress_update_multi_param(4, progress_cols, progress_vals); + } + } StartTransactionCommand(); @@ -3697,7 +3801,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) if (indexRel->rd_rel->relpersistence == RELPERSISTENCE_TEMP) elog(ERROR, "cannot reindex a temporary table concurrently"); - pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, + /* Don't overwrite CREATE INDEX command */ + if (!(params->options & REINDEXOPT_REPORT_CREATE_PART)) + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, idx->tableId); progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; @@ -3857,9 +3963,11 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* * Update progress for the index to build, with the correct parent - * table involved. + * table involved. Don't overwrite CREATE INDEX command. */ - pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, newidx->tableId); + if (!(params->options & REINDEXOPT_REPORT_CREATE_PART)) + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, newidx->tableId); + progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; progress_vals[1] = PROGRESS_CREATEIDX_PHASE_BUILD; progress_vals[2] = newidx->indexId; @@ -3921,10 +4029,12 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) /* * Update progress for the index to build, with the correct parent - * table involved. + * table involved. Don't overwrite CREATE INDEX command. */ - pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, + if (!(params->options & REINDEXOPT_REPORT_CREATE_PART)) + pgstat_progress_start_command(PROGRESS_COMMAND_CREATE_INDEX, newidx->tableId); + progress_vals[0] = PROGRESS_CREATEIDX_COMMAND_REINDEX_CONCURRENTLY; progress_vals[1] = PROGRESS_CREATEIDX_PHASE_VALIDATE_IDXSCAN; progress_vals[2] = newidx->indexId; @@ -4159,7 +4269,9 @@ ReindexRelationConcurrently(Oid relationOid, ReindexParams *params) MemoryContextDelete(private_context); - pgstat_progress_end_command(); + /* Don't overwrite CREATE INDEX command. */ + if (!(params->options & REINDEXOPT_REPORT_CREATE_PART)) + pgstat_progress_end_command(); return true; } diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index 91c28868d47..4142894c230 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -42,6 +42,7 @@ typedef struct ReindexParams #define REINDEXOPT_REPORT_PROGRESS 0x02 /* report pgstat progress */ #define REINDEXOPT_MISSING_OK 0x04 /* skip missing relations */ #define REINDEXOPT_CONCURRENTLY 0x08 /* concurrent mode */ +#define REINDEXOPT_REPORT_CREATE_PART 0x10 /* report that index was created for partition */ /* state info for validate_index bulkdelete callback */ typedef struct ValidateIndexState diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 1bdd430f063..6b2320244b8 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,139 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 3 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart11 + Partitioned table "public.idxpart11" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart1 FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart11_a_idx" btree (a) + "idxpart11_a_idx1" btree (a) + "idxpart11_a_idx2" btree (a) + "idxpart11_a_idx3" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart111 + Partitioned table "public.idxpart111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart11 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart111_a_idx" btree (a) + "idxpart111_a_idx1" btree (a) + "idxpart111_a_idx2" btree (a) + "idxpart111_a_idx3" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart1111 + Partitioned table "public.idxpart1111" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart111 DEFAULT +Partition key: RANGE (a) +Indexes: + "idxpart1111_a_idx" btree (a) + "idxpart1111_a_idx1" btree (a) + "idxpart1111_a_idx2" btree (a) + "idxpart1111_a_idx3" UNIQUE, btree (a) INVALID +Number of partitions: 0 + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + +\d idxpart3 + Partitioned table "public.idxpart3" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (30) TO (40) +Partition key: RANGE (a) +Indexes: + "idxpart3_a_idx" btree (a) + "idxpart3_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart31 + Table "public.idxpart31" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart3 DEFAULT +Indexes: + "idxpart31_a_idx" btree (a) + "idxpart31_a_idx1" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 429120e7104..14e0513386c 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,30 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a); +create table idxpart31 partition of idxpart3 default; + +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart11 +\d idxpart111 +\d idxpart1111 +\d idxpart2 +\d idxpart3 +\d idxpart31 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.25.1 --aPdhxNJGSeOG9wFI-- ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v6 1/2] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 118 ++++++++++++++++++++----- src/test/regress/expected/indexing.out | 60 ++++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 165 insertions(+), 40 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f1b5f87e6a..61d0c4914c 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -665,17 +665,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1110,6 +1099,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initial build of index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1162,6 +1156,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1173,8 +1175,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1217,10 +1221,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1291,10 +1297,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1306,8 +1316,8 @@ DefineIndex(Oid relationId, childStmt->relation = NULL; childStmt->indexOid = InvalidOid; childStmt->oldNode = InvalidOid; - childStmt->oldCreateSubid = InvalidSubTransactionId; - childStmt->oldFirstRelfilenodeSubid = InvalidSubTransactionId; + // childStmt->oldCreateSubid = InvalidSubTransactionId; + // childStmt->oldFirstRelfilenodeSubid = InvalidSubTransactionId; /* * Adjust any Vars (both in expressions and in the index's @@ -1346,10 +1356,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1376,34 +1394,86 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + PopActiveSnapshot(); + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + CommandCounterIncrement(); + PushActiveSnapshot(GetTransactionSnapshot()); + + if (get_index_isvalid(indexrelid)) + { + PopActiveSnapshot(); + continue; + } + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + } + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + CommitTransactionCommand(); + StartTransactionCommand(); + PushActiveSnapshot(GetTransactionSnapshot()); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 7e78a07af8..f6eba3dd40 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JgQwtEuHJzHdouWu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0002-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v7 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 135 +++++++++++++++++-------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 166 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f1b5f87e6a..d417404211 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -91,6 +91,7 @@ static bool ReindexRelationConcurrently(Oid relationOid, int options); static void ReindexPartitions(Oid relid, int options, bool isTopLevel); static void ReindexMultipleInternal(List *relids, int options); +static void reindex_invalid_child_indexes(Oid indexRelationId); static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -665,17 +666,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1110,6 +1100,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1162,6 +1157,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1169,12 +1172,15 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1217,10 +1223,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1291,10 +1299,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1346,10 +1358,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1359,51 +1379,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1585,6 +1596,50 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + List *childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + PopActiveSnapshot(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + CommandCounterIncrement(); + PushActiveSnapshot(GetTransactionSnapshot()); + + if (get_index_isvalid(indexrelid) || + get_rel_relkind(indexrelid) == RELKIND_PARTITIONED_INDEX) + { + PopActiveSnapshot(); + continue; + } + + ReindexRelationConcurrently(indexrelid, 0); + } + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 7e78a07af8..c6d7cfd446 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --MAH+hnPXVZWQ5cD/ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0002-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v8 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 135 +++++++++++++++++-------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 166 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index f1b5f87e6a..d417404211 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -91,6 +91,7 @@ static bool ReindexRelationConcurrently(Oid relationOid, int options); static void ReindexPartitions(Oid relid, int options, bool isTopLevel); static void ReindexMultipleInternal(List *relids, int options); +static void reindex_invalid_child_indexes(Oid indexRelationId); static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -665,17 +666,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1110,6 +1100,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1162,6 +1157,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1169,12 +1172,15 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1217,10 +1223,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1291,10 +1299,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1346,10 +1358,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1359,51 +1379,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1585,6 +1596,50 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + List *childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + PopActiveSnapshot(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + CommandCounterIncrement(); + PushActiveSnapshot(GetTransactionSnapshot()); + + if (get_index_isvalid(indexrelid) || + get_rel_relkind(indexrelid) == RELKIND_PARTITIONED_INDEX) + { + PopActiveSnapshot(); + continue; + } + + ReindexRelationConcurrently(indexrelid, 0); + } + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 7e78a07af8..c6d7cfd446 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --FCuugMFkClbJLl1L Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0002-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v10 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 141 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 172 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 749db2845e..ba4424d379 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -661,15 +661,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 996f1ed070..c1dd4c8362 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -93,6 +93,7 @@ static bool ReindexRelationConcurrently(Oid relationOid, int options); static void ReindexPartitions(Oid relid, int options, bool isTopLevel); static void ReindexMultipleInternal(List *relids, int options); +static void reindex_invalid_child_indexes(Oid indexRelationId); static void reindex_error_callback(void *args); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -667,17 +668,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1111,6 +1101,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1163,6 +1158,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1170,12 +1173,15 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1218,10 +1224,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1292,10 +1300,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1347,10 +1359,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1360,51 +1380,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1586,6 +1597,56 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, REINDEXOPT_CONCURRENTLY); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index 49b6f7c18f..2fb00a2fa7 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index fc1479dca6..610c12de9e 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --i3lJ51RuaGWuFYNw Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0002-Add-SKIPVALID-flag-for-more-integration.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v11 1/9] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. XXX: does pgstat_progress_update_param() break other commands progress ? --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 141 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 172 insertions(+), 56 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 29dee5689e..bd4431a3ce 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -661,15 +661,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index ca24620fd0..76219381c1 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -671,17 +672,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1119,6 +1109,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1171,6 +1166,14 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); @@ -1178,12 +1181,15 @@ DefineIndex(Oid relationId, TupleDesc parentDesc; Oid *opfamOids; + // If concurrent, maybe this should be done after excluding indexes which already exist ? pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1226,10 +1232,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1300,10 +1308,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1355,10 +1367,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1368,51 +1388,42 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + table_close(rel, NoLock); + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries @@ -1606,6 +1617,56 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + */ + ReindexMultipleInternal(partitions, REINDEXOPT_CONCURRENTLY); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --cWoXeonUoKmBZSoM Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0002-Add-SKIPVALID-flag-for-more-integration.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v1] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) --- doc/src/sgml/ref/create_index.sgml | 9 ----- src/backend/commands/indexcmds.c | 54 +++++++++++++++++------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index ff87b2d28f..e1298b8523 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 2baca12c5f..d2809c85a0 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -652,17 +652,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1139,15 +1128,23 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { + char *relname = pstrdup(RelationGetRelationName(rel)); + /* * Unless caller specified to skip this step (via ONLY), process each * partition to make sure they all contain a corresponding index. * * If we're called internally (no stmt->relation), recurse always. */ - if (!stmt->relation || stmt->relation->inh) + if (stmt->relation && !stmt->relation->inh) + table_close(rel, NoLock); + else { PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; @@ -1166,6 +1163,8 @@ DefineIndex(Oid relationId, for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); + table_close(rel, NoLock); + /* * For each partition, scan all existing indexes; if one matches * our index definition and is not already attached to some other @@ -1196,9 +1195,9 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; @@ -1365,21 +1364,35 @@ DefineIndex(Oid relationId, } } + /* + * CIC needs to mark a partitioned table as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + if (concurrent) + { + if (ActiveSnapshotSet()) + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); + } + /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1387,11 +1400,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That -- 2.17.0 --aVD9QWMuhilNxW9f-- ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v2 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) --- doc/src/sgml/ref/create_index.sgml | 9 ---- src/backend/commands/indexcmds.c | 68 ++++++++++++++------------ src/test/regress/expected/indexing.out | 14 +++++- src/test/regress/sql/indexing.sql | 3 +- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index ff87b2d28f..e1298b8523 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 2baca12c5f..344cabaf52 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -652,17 +652,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1139,8 +1128,26 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { + /* + * Need to close the relation before recursing into children, so copy + * needed data. + */ + PartitionDesc partdesc = RelationGetPartitionDesc(rel); + int nparts = partdesc->nparts; + TupleDesc parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + char *relname = pstrdup(RelationGetRelationName(rel)); + Oid *part_oids; + + part_oids = palloc(sizeof(Oid) * nparts); + memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + table_close(rel, NoLock); + /* * Unless caller specified to skip this step (via ONLY), process each * partition to make sure they all contain a corresponding index. @@ -1149,19 +1156,11 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { - PartitionDesc partdesc = RelationGetPartitionDesc(rel); - int nparts = partdesc->nparts; - Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; - TupleDesc parentDesc; Oid *opfamOids; pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_TOTAL, nparts); - - memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); - - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1196,9 +1195,9 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; @@ -1365,21 +1364,35 @@ DefineIndex(Oid relationId, } } + /* + * CIC needs to mark a partitioned table as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + if (concurrent) + { + if (ActiveSnapshotSet()) + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); + } + /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1387,11 +1400,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..17fa77e317 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,21 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); create table idxpart1 partition of idxpart for values from (0) to (10); create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +\d idxpart1 + Table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Indexes: + "idxpart1_a_idx" btree (a) + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..19a21eba9d 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,11 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); create table idxpart1 partition of idxpart for values from (0) to (10); create index concurrently on idxpart (a); +\d idxpart1 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JwB53PgKC5A7+0Ej Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0002-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v3 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 ---- src/backend/commands/indexcmds.c | 63 ++++++++++++++++---------- src/test/regress/expected/indexing.out | 26 +++++++++-- src/test/regress/sql/indexing.sql | 12 +++-- 4 files changed, 71 insertions(+), 39 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index ff87b2d28f..e1298b8523 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 2baca12c5f..42c905867c 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -652,17 +652,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1139,6 +1128,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1149,8 +1142,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1160,8 +1162,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1196,18 +1200,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1333,6 +1339,8 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + PushActiveSnapshot(GetTransactionSnapshot()); } pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, @@ -1363,23 +1371,37 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + /* + * CIC needs to mark a partitioned table as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + if (concurrent) + { + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1387,11 +1409,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..85d3e37ae1 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,29 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) +Number of partitions: 1 (Use \d+ to list them.) + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..56a117d75f 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,16 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +\d idxpart1 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --SCOJXUq1iwCn05li Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v3-0002-Implement-REINDEX-of-partitioned-tables-indexes.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 109 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 ++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 150 insertions(+), 41 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index df3e567acb..291046cb16 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -653,17 +653,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1098,6 +1087,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build index partition as invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1140,6 +1132,10 @@ DefineIndex(Oid relationId, CreateComments(indexRelationId, RelationRelationId, 0, stmt->idxcomment); + /* save lockrelid and locktag for below */ + heaprelid = rel->rd_lockInfo.lockRelId; + SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); + if (partitioned) { /* @@ -1150,8 +1146,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1161,8 +1166,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1197,18 +1204,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1334,10 +1343,16 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1364,23 +1379,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1388,11 +1452,6 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ - heaprelid = rel->rd_lockInfo.lockRelId; - SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); - /* * For a concurrent build, it's important to make the catalog entries * visible to other transactions before we start to build the index. That diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 --- src/backend/commands/indexcmds.c | 104 +++++++++++++++++++------ src/test/regress/expected/indexing.out | 57 +++++++++++++- src/test/regress/sql/indexing.sql | 16 +++- 4 files changed, 148 insertions(+), 38 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index 33aa64e81d..c780dc9547 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -657,15 +657,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 68d75916ae..cdf31a47d8 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -655,17 +655,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1100,6 +1089,9 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + /* Initial concurrent build of index partition is invalid */ + flags |= INDEX_CREATE_INVALID; if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1152,8 +1144,17 @@ DefineIndex(Oid relationId, */ if (!stmt->relation || stmt->relation->inh) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); PartitionDesc partdesc = RelationGetPartitionDesc(rel); int nparts = partdesc->nparts; + char *relname = pstrdup(RelationGetRelationName(rel)); Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; TupleDesc parentDesc; @@ -1163,8 +1164,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1199,18 +1202,20 @@ DefineIndex(Oid relationId, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot create unique index on partitioned table \"%s\"", - RelationGetRelationName(rel)), + relname), errdetail("Table \"%s\" contains partitions that are foreign tables.", - RelationGetRelationName(rel)))); + relname))); table_close(childrel, lockmode); continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1336,10 +1341,17 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1366,23 +1378,72 @@ DefineIndex(Oid relationId, table_close(pg_index, RowExclusiveLock); heap_freetuple(newtup); } + } else + table_close(rel, NoLock); + + if (concurrent) + { + List *childs; + ListCell *lc; + int npart = 0; + + /* Reindex invalid child indexes created earlier */ + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); + childs = find_inheritance_children(indexRelationId, NoLock); + MemoryContextSwitchTo(oldcontext); + + /* Make the catalog changes visible to get_partition_parent */ + CommandCounterIncrement(); + + foreach (lc, childs) + { + Oid indexrelid = lfirst_oid(lc); + Relation cldidx; + bool isvalid; + + if (!OidIsValid(parentIndexId)) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + npart++); + + cldidx = index_open(indexrelid, ShareUpdateExclusiveLock); + isvalid = cldidx->rd_index->indisvalid; + index_close(cldidx, NoLock); + if (isvalid) + continue; + + /* This may be a partitioned index, which is fine too */ + ReindexRelationConcurrently(indexrelid, 0); + PushActiveSnapshot(GetTransactionSnapshot()); + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); } /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); return address; } + table_close(rel, NoLock); if (!concurrent) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); - /* If this is the top-level index, we're done. */ if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1390,10 +1451,9 @@ DefineIndex(Oid relationId, return address; } - /* save lockrelid and locktag for below, then close rel */ + /* save lockrelid and locktag for below */ heaprelid = rel->rd_lockInfo.lockRelId; SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId); - table_close(rel, NoLock); /* * For a concurrent build, it's important to make the catalog entries diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index f78865ef81..1ae58e110f 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,60 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 35d159f41b..d908ee6d17 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,20 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --hYooF8G/hrfVAmum Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0003-Implement-CLUSTER-of-partitioned-table.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table @ 2020-06-06 22:42 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw) Note, this effectively reverts 050098b14, so take care to not reintroduce the bug it fixed. --- doc/src/sgml/ref/create_index.sgml | 9 -- src/backend/commands/indexcmds.c | 143 ++++++++++++++++++------- src/test/regress/expected/indexing.out | 60 ++++++++++- src/test/regress/sql/indexing.sql | 18 +++- 4 files changed, 176 insertions(+), 54 deletions(-) diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml index a5271a9f8f..6869a18968 100644 --- a/doc/src/sgml/ref/create_index.sgml +++ b/doc/src/sgml/ref/create_index.sgml @@ -686,15 +686,6 @@ Indexes: cannot. </para> - <para> - Concurrent builds for indexes on partitioned tables are currently not - supported. However, you may concurrently build the index on each - partition individually and then finally create the partitioned index - non-concurrently in order to reduce the time where writes to the - partitioned table will be locked out. In this case, building the - partitioned index is a metadata only operation. - </para> - </refsect2> </refsect1> diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 127ba7835d..4ac1dacd7d 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -68,6 +68,7 @@ /* non-export function prototypes */ +static void reindex_invalid_child_indexes(Oid indexRelationId); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); static void CheckPredicate(Expr *predicate); static void ComputeIndexAttrs(IndexInfo *indexInfo, @@ -680,17 +681,6 @@ DefineIndex(Oid relationId, partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE; if (partitioned) { - /* - * Note: we check 'stmt->concurrent' rather than 'concurrent', so that - * the error is thrown also for temporary tables. Seems better to be - * consistent, even though we could do it on temporary table because - * we're not actually doing it concurrently. - */ - if (stmt->concurrent) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create index on partitioned table \"%s\" concurrently", - RelationGetRelationName(rel)))); if (stmt->excludeOpNames) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId, if (pd->nparts != 0) flags |= INDEX_CREATE_INVALID; } + else if (concurrent && OidIsValid(parentIndexId)) + { + /* If concurrent, initially build index partitions as "invalid" */ + flags |= INDEX_CREATE_INVALID; + } if (stmt->deferrable) constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE; @@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId, partdesc = RelationGetPartitionDesc(rel); if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0) { + /* + * Need to close the relation before recursing into children, so + * copy needed data into a longlived context. + */ + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext = MemoryContextSwitchTo(ind_context); int nparts = partdesc->nparts; Oid *part_oids = palloc(sizeof(Oid) * nparts); bool invalidate_parent = false; @@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId, nparts); memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts); + parentDesc = CreateTupleDescCopy(RelationGetDescr(rel)); + table_close(rel, NoLock); + MemoryContextSwitchTo(oldcontext); - parentDesc = RelationGetDescr(rel); opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes); for (i = 0; i < numberOfKeyAttributes; i++) opfamOids[i] = get_opclass_family(classObjectId[i]); @@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId, continue; } + oldcontext = MemoryContextSwitchTo(ind_context); childidxs = RelationGetIndexList(childrel); attmap = build_attrmap_by_name(RelationGetDescr(childrel), parentDesc); + MemoryContextSwitchTo(oldcontext); foreach(cell, childidxs) { @@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId, */ if (!found) { - IndexStmt *childStmt = copyObject(stmt); + IndexStmt *childStmt; bool found_whole_row; ListCell *lc; + oldcontext = MemoryContextSwitchTo(ind_context); + childStmt = copyObject(stmt); + MemoryContextSwitchTo(oldcontext); + /* * We can't use the same index name for the child index, * so clear idxname to let the recursive invocation choose @@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId, createdConstraintId, is_alter_table, check_rights, check_not_in_use, skip_build, quiet); + if (concurrent) + { + PopActiveSnapshot(); + PushActiveSnapshot(GetTransactionSnapshot()); + invalidate_parent = true; + } } - pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, - i + 1); + /* For concurrent build, this is a catalog-only stage */ + if (!concurrent) + pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + i + 1); free_attrmap(attmap); } @@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId, * invalid, this is incorrect, so update our row to invalid too. */ if (invalidate_parent) - { - Relation pg_index = table_open(IndexRelationId, RowExclusiveLock); - HeapTuple tup, - newtup; - - tup = SearchSysCache1(INDEXRELID, - ObjectIdGetDatum(indexRelationId)); - if (!HeapTupleIsValid(tup)) - elog(ERROR, "cache lookup failed for index %u", - indexRelationId); - newtup = heap_copytuple(tup); - ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false; - CatalogTupleUpdate(pg_index, &tup->t_self, newtup); - ReleaseSysCache(tup); - table_close(pg_index, RowExclusiveLock); - heap_freetuple(newtup); - } - } + index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID); + } else + table_close(rel, NoLock); /* * Indexes on partitioned tables are not themselves built, so we're * done here. */ - table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) + { + if (concurrent) + reindex_invalid_child_indexes(indexRelationId); + pgstat_progress_end_command(); + } + return address; } - if (!concurrent) + if (!concurrent || OidIsValid(parentIndexId)) { - /* Close the heap and we're done, in the non-concurrent case */ - table_close(rel, NoLock); + /* + * We're done if this is the top-level index, + * or the catalog-only phase of a partition built concurrently + */ - /* If this is the top-level index, we're done. */ + table_close(rel, NoLock); if (!OidIsValid(parentIndexId)) pgstat_progress_end_command(); @@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId, return address; } +/* Reindex invalid child indexes created earlier */ +static void +reindex_invalid_child_indexes(Oid indexRelationId) +{ + ListCell *lc; + int npart = 0; + ReindexParams params = { + .options = REINDEXOPT_CONCURRENTLY + }; + + MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX", + ALLOCSET_DEFAULT_SIZES); + MemoryContext oldcontext; + List *childs = find_inheritance_children(indexRelationId, ShareLock); + List *partitions = NIL; + + PreventInTransactionBlock(true, "REINDEX INDEX"); + + foreach (lc, childs) + { + Oid partoid = lfirst_oid(lc); + + /* XXX: need to retrofit progress reporting into it */ + // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE, + // npart++); + + if (get_index_isvalid(partoid) || + !RELKIND_HAS_STORAGE(get_rel_relkind(partoid))) + continue; + + /* Save partition OID */ + oldcontext = MemoryContextSwitchTo(ind_context); + partitions = lappend_oid(partitions, partoid); + MemoryContextSwitchTo(oldcontext); + } + + /* + * Process each partition listed in a separate transaction. Note that + * this commits and then starts a new transaction immediately. + * XXX: since this is done in 2*N transactions, it could just as well + * call ReindexRelationConcurrently directly + */ + ReindexMultipleInternal(partitions, ¶ms); + + /* + * CIC needs to mark a partitioned index as VALID, which itself + * requires setting READY, which is unset for CIC (even though + * it's meaningless for an index without storage). + * This must be done only while holding a lock which precludes adding + * partitions. + * See also: validatePartitionedIndex(). + */ + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY); + CommandCounterIncrement(); + index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID); +} /* * CheckMutability diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out index c93f4470c9..f04abc6897 100644 --- a/src/test/regress/expected/indexing.out +++ b/src/test/regress/expected/indexing.out @@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass (8 rows) drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); -ERROR: cannot create index on partitioned table "idxpart" concurrently +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +ERROR: could not create unique index "idxpart2_a_idx2_ccnew" +DETAIL: Key (a)=(10) is duplicated. +\d idxpart + Partitioned table "public.idxpart" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition key: RANGE (a) +Indexes: + "idxpart_a_idx" btree (a) + "idxpart_a_idx1" UNIQUE, btree (a) INVALID +Number of partitions: 2 (Use \d+ to list them.) + +\d idxpart1 + Partitioned table "public.idxpart1" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (0) TO (10) +Partition key: RANGE (a) +Indexes: + "idxpart1_a_idx" btree (a) INVALID + "idxpart1_a_idx1" btree (a) + "idxpart1_a_idx2" UNIQUE, btree (a) INVALID +Number of partitions: 1 (Use \d+ to list them.) + +\d idxpart2 + Table "public.idxpart2" + Column | Type | Collation | Nullable | Default +--------+---------+-----------+----------+--------- + a | integer | | | + b | integer | | | + c | text | | | +Partition of: idxpart FOR VALUES FROM (10) TO (20) +Indexes: + "idxpart2_a_idx" btree (a) + "idxpart2_a_idx1" btree (a) + "idxpart2_a_idx2" UNIQUE, btree (a) INVALID + "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID + drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- https://postgr.es/m/[email protected] diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql index 42f398b67c..3d4b6e9bc9 100644 --- a/src/test/regress/sql/indexing.sql +++ b/src/test/regress/sql/indexing.sql @@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass where relname like 'idxpart%' order by relname; drop table idxpart; --- Some unsupported features +-- CIC on partitioned table create table idxpart (a int, b int, c text) partition by range (a); -create table idxpart1 partition of idxpart for values from (0) to (10); -create index concurrently on idxpart (a); +create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a); +create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a); +create table idxpart111 partition of idxpart11 default partition by range(a); +create table idxpart1111 partition of idxpart111 default partition by range(a); +create table idxpart2 partition of idxpart for values from (10) to (20); +insert into idxpart2 values(10),(10); -- not unique +create index concurrently on idxpart (a); -- partitioned +create index concurrently on idxpart1 (a); -- partitioned and partition +create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves +create index concurrently on idxpart2 (a); -- leaf +create unique index concurrently on idxpart (a); -- partitioned, unique failure +\d idxpart +\d idxpart1 +\d idxpart2 drop table idxpart; -- Verify bugfix with query on indexed partitioned table with no partitions -- 2.17.0 --QTprm0S8XgL7H0Dt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-f-progress-reporting.patch" ^ permalink raw reply [nested|flat] 80+ messages in thread
* libpq compression (part 3) @ 2023-12-19 16:40 Jacob Burroughs <[email protected]> 0 siblings, 2 replies; 80+ messages in thread From: Jacob Burroughs @ 2023-12-19 16:40 UTC (permalink / raw) To: [email protected] Hello PG developers! I would like to introduce an updated take on libpq protocol-level compression, building off off the work in https://www.postgresql.org/message-id/[email protected] and the followon work in https://www.postgresql.org/message-id/[email protected] along with all of the (nice and detailed) feedback and discussion therein. The first patch in the stack replaces `secure_read` and `secure_write` (and their frontend counterparts) with an "IO stream" abstraction, which should address a lot of the concerns from all parties around secure_read. The fundamental idea of an IO stream is that it is a linked list of "IoStreamProcessor"s. The "base" processor is the actual socket-backed API, and then SSL, GSSAPI, and compression can all add layers on top of that base that add functionality and rely on the layer below to read/write data. This structure makes it easy to add compression on top of either plain or encrypted sockets through a unified, unconditional API, and also makes it easy for callers to use plain, plain-compressed, secure, and secure-compressed communication channels equivalently. The second patch is the refactored implementation of compression itself, with ZSTD support merged into the main patch because the configuration-level work is now already merged in master. There was a good bit of rebasing, housekeeping, and bugfixing (including fixing lz4 by making it now be explicitly buffered inside ZStream), along with taking into account a lot of the feedback from this mailing list. I reworked the API to use the general compression processing types and methods from `common/compression`. This change also refactors the protocol to require the minimum amount of new message types and exchanges possible, while also enabling one-directional compression. The compression "handshaking" process now looks as follows: 1. Client sends startup packet with `_pq_.libpq_compression = alg1;alg2` 2. At this point, the server can immediately begin compressing packets to the client with any of the specified algorithms it supports if it so chooses 3. Server includes `libpq_compression` in the automatically sent `ParameterStatus` messages before handshaking 4. At this point, the client can immediately begin compressing packets to the server with any of the supported algorithms Both the server and client will prefer to compress using the first algorithm in their list that the other side supports, and we explicitly support `none` in the algorithm list. This allows e.g. a client to use `none;gzip` and a server to use `zstd;gzip;lz4`, and then the client will not compress its data but the server will send its data using gzip. Each side uses its own configured compression level (if set), since compression levels affect compression effort much more than decompression effort. This change also allows connections to succeed if compression was requested but not available (most of the time, I imagine that a client would prefer to just not use compression if the server doesn't support it; unlike SSL, it's a nice to have not an essential. If a client application actually really *needs* compression, that can still be facilitated by explicitly checking the negotiated compression methods.) The third patch adds the traffic monitoring statistics that had been in the main patch of the previous series. I've renamed them and changed slightly where to measure the actual raw network bytes and the "logical" protocol bytes, which also means this view can measure SSL/GSSAPI overhead (not that that's a particularly *important* thing to measure, but it's worth nothing what the view will actually measure. The fourth patch adds a TAP test that validates all of the compression methods and compression negotiation. Ideally it would probably be part of patch #2, but it uses the monitoring from #3 to be able to validate that compression is actually working. The fifth patch is just a placeholder to allow running the test suite with compression maximally enabled to work out any kinks. I believe this patch series is ready for detailed review/testing, with one caveat: as can be seen here https://cirrus-ci.com/build/6732518292979712 , the build is passing on all platforms and all tests except for the primary SSL test on Windows. After some network-level debugging, it appears that we are bumping into a version of the issues seen here https://www.postgresql.org/message-id/flat/CA%2BhUKG%2BOeoETZQ%3DQw5Ub5h3tmwQhBmDA%3DnuNO3KG%3DzWfUy... , where on Windows some SSL error messages end up getting swallowed up by the the process exiting and closing the socket with a RST rather than a nice clean shutdown. I may have the cause/effect wrong here, but the issues appear before the compression is actually fully set up in the client used and would appear to be a side effect of timing differences and/or possibly size differences in the startup packet. Any pointers on how to resolve this would be appreciated. It does reproduce on Windows fairly readily, though any one particular test still sometimes succeeds, and the relevant SSL connection failure message reliably shows up in Wireshark. Also please let me know if I have made any notable mailing list/patch etiquette/format/structure errors. This is my first time submitting a patch to a mailing-list driven open source project and while I have tried to carefully review the various wiki guides I'm sure I didn't take everything in perfectly. Thanks, Jacob Burroughs Attachments: [application/octet-stream] v1-0004-Add-basic-test-for-compression-functionality.patch (6.9K, ../../CACzsqT4cJG0kaCbz24Sd=GAEgiQDpzU8yuD6vF25zo870+3M6g@mail.gmail.com/2-v1-0004-Add-basic-test-for-compression-functionality.patch) download | inline diff: From 4496401cfa5c4629d5d1fae568d9fd54d7ff5dc7 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Mon, 18 Dec 2023 14:15:18 -0600 Subject: [PATCH v1 4/5] Add basic test for compression functionality --- src/Makefile.global.in | 2 + src/interfaces/libpq/Makefile | 7 +- src/interfaces/libpq/meson.build | 8 +- src/interfaces/libpq/t/005_compression.pl | 95 +++++++++++++++++++++++ 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/interfaces/libpq/t/005_compression.pl diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 104e5de0fe..ab21065b72 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -195,9 +195,11 @@ with_ldap = @with_ldap@ with_libxml = @with_libxml@ with_libxslt = @with_libxslt@ with_llvm = @with_llvm@ +with_lz4 = @with_lz4@ with_system_tzdata = @with_system_tzdata@ with_uuid = @with_uuid@ with_zlib = @with_zlib@ +with_zstd = @with_zstd@ enable_rpath = @enable_rpath@ enable_nls = @enable_nls@ enable_debug = @enable_debug@ diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index d0a66cfaa0..b0bcbffd66 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -14,6 +14,9 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global export with_ssl +export with_zlib +export with_lz4 +export with_zstd PGFILEDESC = "PostgreSQL Access Library" @@ -81,9 +84,9 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif ifeq ($(PORTNAME), win32) SHLIB_LINK += -lshell32 -lws2_32 -lsecur32 $(filter -leay32 -lssleay32 -lcomerr32 -lkrb5_32, $(LIBS)) diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build index 80e6a15adf..e9b3c22a52 100644 --- a/src/interfaces/libpq/meson.build +++ b/src/interfaces/libpq/meson.build @@ -118,8 +118,14 @@ tests += { 't/002_api.pl', 't/003_load_balance_host_list.pl', 't/004_load_balance_dns.pl', + 't/005_compression.pl', ], - 'env': {'with_ssl': ssl_library}, + 'env': { + 'with_ssl': ssl_library, + 'with_zlib': zlib.found() ? 'yes' : 'no', + 'with_lz4': lz4.found() ? 'yes' : 'no', + 'with_zstd': zstd.found() ? 'yes' : 'no', + }, }, } diff --git a/src/interfaces/libpq/t/005_compression.pl b/src/interfaces/libpq/t/005_compression.pl new file mode 100644 index 0000000000..f029123d2b --- /dev/null +++ b/src/interfaces/libpq/t/005_compression.pl @@ -0,0 +1,95 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group +use strict; +use warnings; +use Config; +use PostgreSQL::Test::Utils; +use PostgreSQL::Test::Cluster; +use Test::More; + +my $withZlib = $ENV{with_zlib} eq 'yes'; +my $withLz4 = $ENV{with_lz4} eq 'yes'; +my $withZstd = $ENV{with_zstd} eq 'yes'; +if (!$withZlib && !$withLz4 && !$withZstd) +{ + plan skip_all => 'no compression methods available'; +} + +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init; +$node->append_conf('postgresql.conf', "libpq_compression = off"); +$node->start; + +# Use a string that any reasonable compression algorithm will compress +my $compressableString = "a" x 1000; + +my $result; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=on"); +is( (split "\n", $result)[-1], "f|f", 'successfully does not compress if server-side compression is disabled'); + +$node->append_conf('postgresql.conf', "libpq_compression = on"); +$node->reload; + +my $result; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=on"); +is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with default algorithms'); + +my $test_algorithm; +if ($withZlib) +{ + $test_algorithm = "gzip"; +} +elsif($withLz4) +{ + $test_algorithm = "lz4"; +} +elsif($withZstd) +{ + $test_algorithm = "zstd"; +} +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=none;$test_algorithm"); +is( (split "\n", $result)[-1], "f|t", 'successfully compresses unidirectionally with none as first client algorithm'); + +$node->append_conf('postgresql.conf', "libpq_compression = 'none;$test_algorithm'"); +$node->reload; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + "compression=on"); +is( (split "\n", $result)[-1], "t|f", 'successfully compresses unidirectionally with none as first server algorithm'); + +$node->append_conf('postgresql.conf', 'libpq_compression = on'); +$node->reload; + +SKIP: { + skip "gzip not available", 1 unless $ENV{with_zlib}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=gzip"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with gzip'); +} + +SKIP: { + skip "lz4 not available", 1 unless $ENV{with_lz4}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=lz4"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with lz4'); +} + +SKIP: { + skip "zstd not available", 1 unless $ENV{with_zstd}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=zstd"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with zstd'); +} + +done_testing(); -- 2.42.0 [application/octet-stream] v1-0003-Add-network-traffic-stats.patch (23.4K, ../../CACzsqT4cJG0kaCbz24Sd=GAEgiQDpzU8yuD6vF25zo870+3M6g@mail.gmail.com/3-v1-0003-Add-network-traffic-stats.patch) download | inline diff: From df7a54c34a585a624b0ec404c278f12cd7252db8 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Fri, 15 Dec 2023 13:08:55 -0600 Subject: [PATCH v1 3/5] Add network traffic stats Adds pg_stat_network_traffic to monitor both bytes sent over the wire and "logical" protocol bytes, allowing for measurement of compression ratio (and SSL overhead) --- doc/src/sgml/monitoring.sgml | 100 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 10 ++ src/backend/libpq/pqcomm.c | 8 ++ src/backend/utils/activity/backend_status.c | 86 +++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 50 +++++++++- src/common/zpq_stream.c | 12 +++ src/include/catalog/pg_proc.dat | 14 ++- src/include/utils/backend_status.h | 10 ++ src/test/regress/expected/rules.out | 15 ++- 9 files changed, 297 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 42509042ad..21ec279d0f 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -350,6 +350,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_stat_network_traffic</structname><indexterm><primary>pg_stat_network_traffic</primary></indexterm></entry> + <entry>One row per connection (regular and replication), showing information about + network traffic of this connection. + See <link linkend="monitoring-pg-stat-network-traffic-view"> + <structname>pg_stat_network_traffic</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_ssl</structname><indexterm><primary>pg_stat_ssl</primary></indexterm></entry> <entry>One row per connection (regular and replication), showing information about @@ -2176,6 +2185,97 @@ description | Waiting for a newly initialized WAL file to reach durable storage </sect2> + <sect2 id="monitoring-pg-stat-network-traffic-view"> + <title><structname>pg_stat_network_traffic</structname></title> + + <indexterm> + <primary>pg_stat_network_traffic</primary> + </indexterm> + + <para> + The <structname>pg_stat_network_traffic</structname> view will contain one row per + backend or WAL sender process, showing statistics about network traffic on + this connection. It can be joined to <structname>pg_stat_activity</structname> + or <structname>pg_stat_replication</structname> on the + <structfield>pid</structfield> column to get more details about the + connection. + </para> + + <table id="pg-stat-network-traffic-view" xreflabel="pg_stat_network_traffic"> + <title><structname>pg_stat_network_traffic</structname> View</title> + <tgroup cols="1"> + <thead> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + Column Type + </para> + <para> + Description + </para></entry> + </row> + </thead> + + <tbody> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>pid</structfield> <type>integer</type> + </para> + <para> + Process ID of a backend or WAL sender process + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>rx_socket_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes received from the underlying socket of this connection + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>tx_socket_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes sent to the underlying socket of this connection + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>rx_pq_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes of raw libpq traffic recieved from this connection. + This can be used to estimate the overhead of SSL, which will generally + make this number smaller than <structfield>rx_socket_bytes</structfield>, + and to estimate the compression ratio when using protocol compression, + as compression will generally make this number larger than + <structfield>rx_socket_bytes</structfield>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>tx_pq_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes of raw libpq traffic sent through this connection. + This can be used to estimate the overhead of SSL, which will generally + make this number smaller than <structfield>tx_socket_bytes</structfield>, + and to estimate the compression ratio when using protocol compression, + as compression will generally make this number larger than + <structfield>tx_socket_bytes</structfield>. + </para></entry> + </row> + </tbody> + </tgroup> + </table> + + </sect2> + <sect2 id="monitoring-pg-stat-ssl-view"> <title><structname>pg_stat_ssl</structname></title> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 11d18ed9dd..b71234b9b7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -893,6 +893,16 @@ CREATE VIEW pg_stat_activity AS LEFT JOIN pg_database AS D ON (S.datid = D.oid) LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); +CREATE VIEW pg_stat_network_traffic AS + SELECT + S.pid, + S.rx_socket_bytes, + S.tx_socket_bytes, + S.rx_pq_bytes, + S.tx_pq_bytes + FROM pg_stat_get_activity(NULL) AS S + WHERE S.client_port IS NOT NULL; + CREATE VIEW pg_stat_replication AS SELECT S.pid, diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 60363ec1e6..78f515f43c 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -966,6 +966,8 @@ socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffer #ifdef WIN32 pgwin32_noblock = false; #endif + if (n > 0) + pgstat_report_rx_socket_traffic(n); return n; } @@ -982,6 +984,8 @@ socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size #ifdef WIN32 pgwin32_noblock = false; #endif + if (n > 0) + pgstat_report_tx_socket_traffic(n); if (n >= 0) { @@ -1009,6 +1013,8 @@ io_read_with_wait(Port *port, void *ptr, size_t len) retry: port->waitfor = WL_SOCKET_READABLE; n = io_stream_read(port->io_stream, ptr, len, false); + if (n > 0) + pgstat_report_rx_pq_traffic(n); /* In blocking mode, wait until the socket is ready */ if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) @@ -1092,6 +1098,8 @@ retry: */ rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count); *bytes_processed += count; + if (count > 0) + pgstat_report_tx_pq_traffic(count); if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) { diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 6e734c6caf..c5712d86b8 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -338,6 +338,9 @@ pgstat_bestart(void) lbeentry.st_xact_start_timestamp = 0; lbeentry.st_databaseid = MyDatabaseId; + lbeentry.st_rx_socket_bytes = lbeentry.st_tx_socket_bytes = + lbeentry.st_rx_pq_bytes = lbeentry.st_tx_pq_bytes = 0; + /* We have userid for client-backends, wal-sender and bgworker processes */ if (lbeentry.st_backendType == B_BACKEND || lbeentry.st_backendType == B_WAL_SENDER @@ -717,6 +720,89 @@ pgstat_report_xact_timestamp(TimestampTz tstamp) PGSTAT_END_WRITE_ACTIVITY(beentry); } +/* + * Report network raw or compressed tx/rx traffic as the specified values. + */ +void +pgstat_report_rx_socket_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_rx_socket_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_tx_socket_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_tx_socket_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_rx_pq_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_rx_pq_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_tx_pq_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_tx_pq_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + /* ---------- * pgstat_read_current_status() - * diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 0cea320c00..936a2d81c3 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -304,7 +304,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) Datum pg_stat_get_activity(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_ACTIVITY_COLS 31 +#define PG_STAT_GET_ACTIVITY_COLS 35 int num_backends = pgstat_fetch_stat_numbackends(); int curr_backend; int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0); @@ -617,6 +617,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[30] = true; else values[30] = UInt64GetDatum(beentry->st_query_id); + values[31] = UInt64GetDatum(beentry->st_rx_socket_bytes); + values[32] = UInt64GetDatum(beentry->st_tx_socket_bytes); + values[33] = UInt64GetDatum(beentry->st_rx_pq_bytes); + values[34] = UInt64GetDatum(beentry->st_tx_pq_bytes); } else { @@ -646,6 +650,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[28] = true; nulls[29] = true; nulls[30] = true; + nulls[31] = true; + nulls[32] = true; + nulls[33] = true; + nulls[34] = true; } tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -876,6 +884,46 @@ pg_stat_get_backend_start(PG_FUNCTION_ARGS) PG_RETURN_TIMESTAMPTZ(result); } +Datum +pg_stat_get_network_traffic(PG_FUNCTION_ARGS) +{ +#define PG_STAT_NETWORK_TRAFFIC_COLS 4 + TupleDesc tupdesc; + Datum values[PG_STAT_NETWORK_TRAFFIC_COLS]; + bool nulls[PG_STAT_NETWORK_TRAFFIC_COLS]; + int32 beid = PG_GETARG_INT32(0); + PgBackendStatus *beentry; + + if ((beentry = pgstat_get_beentry_by_backend_id(beid)) == NULL) + PG_RETURN_NULL(); + else if (!HAS_PGSTAT_PERMISSIONS(beentry->st_userid)) + PG_RETURN_NULL(); + + /* Initialise values and NULL flags arrays */ + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_NETWORK_TRAFFIC_COLS); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "rx_socket_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tx_socket_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "rx_pq_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "tx_pq_bytes", + INT8OID, -1, 0); + BlessTupleDesc(tupdesc); + + /* Fill values and NULLs */ + values[0] = UInt64GetDatum(beentry->st_rx_socket_bytes); + values[1] = UInt64GetDatum(beentry->st_tx_socket_bytes); + values[2] = UInt64GetDatum(beentry->st_rx_pq_bytes); + values[3] = UInt64GetDatum(beentry->st_tx_pq_bytes); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} Datum pg_stat_get_backend_client_addr(PG_FUNCTION_ARGS) diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c index 222a776933..98df28eb3d 100644 --- a/src/common/zpq_stream.c +++ b/src/common/zpq_stream.c @@ -541,6 +541,18 @@ zpq_write(IoStreamLayer * self, ZpqStream * zpq, void const *src, size_t src_siz } else { + /* + * Clear internal buffers on permanent failure, as upper layers will flush + * and tracking of msg_type/msg_len will otherwise get out of sync + */ + if (errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) + { + zpq->tx_msg_bytes_left = 0; + zpq->tx_in.pos = 0; + zpq->tx_in.size = 0; + zpq->tx_out.pos = 0; + zpq->tx_out.size = 0; + } zpq_buf_reuse(&zpq->tx_out); return rc; } diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 77e8b13764..6a1a122d79 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5440,9 +5440,9 @@ proname => 'pg_stat_get_activity', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'int4', - proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id}', + proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}', prosrc => 'pg_stat_get_activity' }, { oid => '8403', descr => 'describe wait events', proname => 'pg_get_wait_events', procost => '10', prorows => '250', @@ -5782,6 +5782,14 @@ proargnames => '{stats_reset,prefetch,hit,skip_init,skip_new,skip_fpw,skip_rep,wal_distance,block_distance,io_depth}', prosrc => 'pg_stat_get_recovery_prefetch' }, +{ oid => '9598', descr => 'statistics: information about network traffic', + proname => 'pg_stat_get_network_traffic', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'record', proargtypes => 'int4', + proallargtypes => '{int4,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o}', + proargnames => '{_beid,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}', + prosrc => 'pg_stat_get_network_traffic' }, + { oid => '2306', descr => 'statistics: information about SLRU caches', proname => 'pg_stat_get_slru', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h index 75fc18c432..5df7bbd797 100644 --- a/src/include/utils/backend_status.h +++ b/src/include/utils/backend_status.h @@ -146,6 +146,12 @@ typedef struct PgBackendStatus /* application name; MUST be null-terminated */ char *st_appname; + /* client-server traffic information */ + uint64 st_rx_socket_bytes; + uint64 st_tx_socket_bytes; + uint64 st_rx_pq_bytes; + uint64 st_tx_pq_bytes; + /* * Current command string; MUST be null-terminated. Note that this string * possibly is truncated in the middle of a multi-byte character. As @@ -321,6 +327,10 @@ extern void pgstat_report_query_id(uint64 query_id, bool force); extern void pgstat_report_tempfile(size_t filesize); extern void pgstat_report_appname(const char *appname); extern void pgstat_report_xact_timestamp(TimestampTz tstamp); +extern void pgstat_report_rx_socket_traffic(uint64 bytes); +extern void pgstat_report_tx_socket_traffic(uint64 bytes); +extern void pgstat_report_rx_pq_traffic(uint64 bytes); +extern void pgstat_report_tx_pq_traffic(uint64 bytes); extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser); extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer, int buflen); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 05070393b9..4b50d7e9f7 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1760,7 +1760,7 @@ pg_stat_activity| SELECT s.datid, s.query_id, s.query, s.backend_type - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) LEFT JOIN pg_database d ON ((s.datid = d.oid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_all_indexes| SELECT c.oid AS relid, @@ -1877,7 +1877,7 @@ pg_stat_gssapi| SELECT pid, gss_princ AS principal, gss_enc AS encrypted, gss_delegation AS credentials_delegated - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) WHERE (client_port IS NOT NULL); pg_stat_io| SELECT backend_type, object, @@ -1898,6 +1898,13 @@ pg_stat_io| SELECT backend_type, fsync_time, stats_reset FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); +pg_stat_network_traffic| SELECT pid, + rx_socket_bytes, + tx_socket_bytes, + rx_pq_bytes, + tx_pq_bytes + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) + WHERE (client_port IS NOT NULL); pg_stat_progress_analyze| SELECT s.pid, s.datid, d.datname, @@ -2079,7 +2086,7 @@ pg_stat_replication| SELECT s.pid, w.sync_priority, w.sync_state, w.reply_time - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_replication_slots| SELECT s.slot_name, @@ -2113,7 +2120,7 @@ pg_stat_ssl| SELECT pid, ssl_client_dn AS client_dn, ssl_client_serial AS client_serial, ssl_issuer_dn AS issuer_dn - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) WHERE (client_port IS NOT NULL); pg_stat_subscription| SELECT su.oid AS subid, su.subname, -- 2.42.0 [application/octet-stream] v1-0001-Add-IO-stream-abstraction.patch (80.3K, ../../CACzsqT4cJG0kaCbz24Sd=GAEgiQDpzU8yuD6vF25zo870+3M6g@mail.gmail.com/4-v1-0001-Add-IO-stream-abstraction.patch) download | inline diff: From 55a8fb38b927f4bf092136afe7230463c7b6bfc2 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Tue, 12 Dec 2023 16:28:03 -0600 Subject: [PATCH v1 1/5] Add IO stream abstraction This is a shared abstraction between the frontend and backend that is designed to draw a clean line between the compoments of postgres that want to consume "regular" protocol bytes and any layers that may want to transform them. In this patch, encyryption is simply moved to use this abstraction rather than callers directly using secure_read/secure_write (which depending on context might or might not actually be secure). This is a pre-work change intended to make compression integrate into frontend and backend comms layers more cleanly. This may be a first step towards facilitating greater sharing of SSL/GSS api code between the frontend and backend down the road, but for now it the stream processors themselves are completely independent. --- src/backend/libpq/be-secure-gssapi.c | 63 ++-- src/backend/libpq/be-secure-openssl.c | 75 +++-- src/backend/libpq/be-secure.c | 216 ------------ src/backend/libpq/pqcomm.c | 245 +++++++++++++- src/backend/postmaster/postmaster.c | 5 + src/common/Makefile | 1 + src/common/io_stream.c | 148 ++++++++ src/common/meson.build | 1 + src/include/common/io_stream.h | 131 ++++++++ src/include/libpq/libpq-be.h | 23 +- src/include/libpq/libpq.h | 6 +- src/interfaces/libpq/fe-connect.c | 343 ++++++++++++++++++- src/interfaces/libpq/fe-misc.c | 31 +- src/interfaces/libpq/fe-secure-gssapi.c | 64 ++-- src/interfaces/libpq/fe-secure-openssl.c | 68 +++- src/interfaces/libpq/fe-secure.c | 411 ----------------------- src/interfaces/libpq/libpq-int.h | 42 +-- src/tools/msvc/Mkvcbuild.pm | 2 +- 18 files changed, 1070 insertions(+), 805 deletions(-) create mode 100644 src/common/io_stream.c create mode 100644 src/include/common/io_stream.h diff --git a/src/backend/libpq/be-secure-gssapi.c b/src/backend/libpq/be-secure-gssapi.c index 8ed2f65347..597ddbae6e 100644 --- a/src/backend/libpq/be-secure-gssapi.c +++ b/src/backend/libpq/be-secure-gssapi.c @@ -74,6 +74,12 @@ static int PqGSSResultNext; /* Next index to read a byte from static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the * results into our output buffer */ +static ssize_t be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +IoStreamProcessor be_gssapi_processor = { + .read = (io_stream_read_func) be_gssapi_read, + .write = (io_stream_write_func) be_gssapi_write +}; /* * Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection. @@ -91,8 +97,8 @@ static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the * recursion. Instead, use elog(COMMERROR) to log extra info about the * failure if necessary, and then return an errno indicating connection loss. */ -ssize_t -be_gssapi_write(Port *port, void *ptr, size_t len) +static int +be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written) { OM_uint32 major, minor; @@ -102,6 +108,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) size_t bytes_encrypted; gss_ctx_id_t gctx = port->gss->ctx; + *bytes_written = 0; + /* * When we get a retryable failure, we must not tell the caller we have * successfully transmitted everything, else it won't retry. For @@ -148,20 +156,21 @@ be_gssapi_write(Port *port, void *ptr, size_t len) */ if (PqGSSSendLength) { - ssize_t ret; + int retval; + size_t count; ssize_t amount = PqGSSSendLength - PqGSSSendNext; - ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext, amount); - if (ret <= 0) - return ret; + retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count); + if (retval < 0 || count == 0) + return retval; /* * Check if this was a partial write, and if so, move forward that * far in our buffer and try again. */ - if (ret < amount) + if (count < amount) { - PqGSSSendNext += ret; + PqGSSSendNext += count; continue; } @@ -242,7 +251,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) /* We're reporting all the data as sent, so reset PqGSSSendConsumed. */ PqGSSSendConsumed = 0; - return bytes_encrypted; + *bytes_written = bytes_encrypted; + return 0; } /* @@ -258,8 +268,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) * We treat fatal errors the same as in be_gssapi_write(), even though the * argument about infinite recursion doesn't apply here. */ -ssize_t -be_gssapi_read(Port *port, void *ptr, size_t len) +static ssize_t +be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) { OM_uint32 major, minor; @@ -269,6 +279,9 @@ be_gssapi_read(Port *port, void *ptr, size_t len) size_t bytes_returned = 0; gss_ctx_id_t gctx = port->gss->ctx; + if (buffered_only) + return false; + /* * The plan here is to read one incoming encrypted packet into * PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out @@ -325,10 +338,10 @@ be_gssapi_read(Port *port, void *ptr, size_t len) /* Collect the length if we haven't already */ if (PqGSSRecvLength < sizeof(uint32)) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, - sizeof(uint32) - PqGSSRecvLength); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + sizeof(uint32) - PqGSSRecvLength, false); - /* If ret <= 0, secure_raw_read already set the correct errno */ + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -359,8 +372,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len) * Read as much of the packet as we are able to on this call into * wherever we left off from the last time we were called. */ - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, - input.length - (PqGSSRecvLength - sizeof(uint32))); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + input.length - (PqGSSRecvLength - sizeof(uint32)), false); /* If ret <= 0, secure_raw_read already set the correct errno */ if (ret <= 0) return ret; @@ -413,7 +426,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len) /* * Read the specified number of bytes off the wire, waiting using - * WaitLatchOrSocket if we would block. + * WaitLatchOrSocket if we would block. Only used during connecition setup + * before GSS is added to the io_stream. * * Results are read into PqGSSRecvBuffer. * @@ -430,7 +444,7 @@ read_or_wait(Port *port, ssize_t len) */ while (PqGSSRecvLength < len) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength); + ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false); /* * If we got back an error and it wasn't just @@ -465,7 +479,7 @@ read_or_wait(Port *port, ssize_t len) */ if (ret == 0) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength); + ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false); if (ret == 0) return -1; } @@ -648,8 +662,10 @@ secure_open_gssapi(Port *port) while (PqGSSSendNext < PqGSSSendLength) { - ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext, - PqGSSSendLength - PqGSSSendNext); + size_t count; + + ret = io_stream_write(port->io_stream, PqGSSSendBuffer + PqGSSSendNext, + PqGSSSendLength - PqGSSSendNext, &count); /* * If we got back an error and it wasn't just @@ -663,7 +679,7 @@ secure_open_gssapi(Port *port) } /* Wait and retry if we couldn't write yet */ - if (ret <= 0) + if (ret < 0 || count == 0) { WaitLatchOrSocket(MyLatch, WL_SOCKET_WRITEABLE | WL_EXIT_ON_PM_DEATH, @@ -671,7 +687,7 @@ secure_open_gssapi(Port *port) continue; } - PqGSSSendNext += ret; + PqGSSSendNext += count; } /* Done sending the packet, reset our buffer */ @@ -701,6 +717,7 @@ secure_open_gssapi(Port *port) pg_GSS_error(_("GSSAPI size check error"), major, minor); return -1; } + io_stream_add_layer(port->io_stream, &be_gssapi_processor, port); port->gss->enc = true; diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 22e3dc5a81..5c67fd46aa 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -59,7 +59,7 @@ openssl_tls_init_hook_typ openssl_tls_init_hook = default_openssl_tls_init; static int my_sock_read(BIO *h, char *buf, int size); static int my_sock_write(BIO *h, const char *buf, int size); static BIO_METHOD *my_BIO_s_socket(void); -static int my_SSL_set_fd(Port *port, int fd); +static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer); static DH *load_dh_file(char *filename, bool isServerStart); static DH *load_dh_buffer(const char *buffer, size_t len); @@ -71,6 +71,16 @@ static bool initialize_dh(SSL_CTX *context, bool isServerStart); static bool initialize_ecdh(SSL_CTX *context, bool isServerStart); static const char *SSLerrmessage(unsigned long ecode); +static ssize_t be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +static void be_tls_close(Port *port); + +IoStreamProcessor be_tls_processor = { + .read = (io_stream_read_func) be_tls_read, + .write = (io_stream_write_func) be_tls_write, + .destroy = (io_stream_destroy_func) be_tls_close +}; + static char *X509_NAME_to_cstring(X509_NAME *name); static SSL_CTX *SSL_context = NULL; @@ -440,7 +450,8 @@ be_tls_open_server(Port *port) SSLerrmessage(ERR_get_error())))); return -1; } - if (!my_SSL_set_fd(port, port->sock)) + if (!my_SSL_set_fd(port->ssl, port->sock, + io_stream_add_layer(port->io_stream, &be_tls_processor, port))) { ereport(COMMERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -674,7 +685,7 @@ aloop: return 0; } -void +static void be_tls_close(Port *port) { if (port->ssl) @@ -704,14 +715,27 @@ be_tls_close(Port *port) } } -ssize_t -be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) +static ssize_t +be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) { ssize_t n; int err; unsigned long ecode; + port->waitfor = 0; errno = 0; + + if (buffered_only) + { + /* + * SSL_pending bytes are guaranteed to be available and readable + * without blocking + */ + len = Min(len, SSL_pending(port->ssl)); + if (len == 0) + return 0; + } + ERR_clear_error(); n = SSL_read(port->ssl, ptr, len); err = SSL_get_error(port->ssl, n); @@ -722,12 +746,12 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) /* a-ok */ break; case SSL_ERROR_WANT_READ: - *waitfor = WL_SOCKET_READABLE; + port->waitfor = WL_SOCKET_READABLE; errno = EWOULDBLOCK; n = -1; break; case SSL_ERROR_WANT_WRITE: - *waitfor = WL_SOCKET_WRITEABLE; + port->waitfor = WL_SOCKET_WRITEABLE; errno = EWOULDBLOCK; n = -1; break; @@ -763,13 +787,14 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) return n; } -ssize_t -be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) +static int +be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written) { ssize_t n; int err; unsigned long ecode; + port->waitfor = 0; errno = 0; ERR_clear_error(); n = SSL_write(port->ssl, ptr, len); @@ -781,12 +806,12 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) /* a-ok */ break; case SSL_ERROR_WANT_READ: - *waitfor = WL_SOCKET_READABLE; + port->waitfor = WL_SOCKET_READABLE; errno = EWOULDBLOCK; n = -1; break; case SSL_ERROR_WANT_WRITE: - *waitfor = WL_SOCKET_WRITEABLE; + port->waitfor = WL_SOCKET_WRITEABLE; errno = EWOULDBLOCK; n = -1; break; @@ -830,7 +855,16 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) break; } - return n; + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } } /* ------------------------------------------------------------ */ @@ -858,7 +892,7 @@ my_sock_read(BIO *h, char *buf, int size) if (buf != NULL) { - res = secure_raw_read(((Port *) BIO_get_app_data(h)), buf, size); + res = io_stream_next_read(BIO_get_app_data(h), buf, size, false); BIO_clear_retry_flags(h); if (res <= 0) { @@ -876,11 +910,12 @@ my_sock_read(BIO *h, char *buf, int size) static int my_sock_write(BIO *h, const char *buf, int size) { - int res = 0; + int res; + size_t bytes_written; - res = secure_raw_write(((Port *) BIO_get_app_data(h)), buf, size); + res = io_stream_next_write(BIO_get_app_data(h), buf, size, &bytes_written); BIO_clear_retry_flags(h); - if (res <= 0) + if (res < 0 || bytes_written == 0) { /* If we were interrupted, tell caller to retry */ if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) @@ -889,7 +924,7 @@ my_sock_write(BIO *h, const char *buf, int size) } } - return res; + return res ? res : bytes_written; } static BIO_METHOD * @@ -935,7 +970,7 @@ my_BIO_s_socket(void) /* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */ static int -my_SSL_set_fd(Port *port, int fd) +my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer) { int ret = 0; BIO *bio; @@ -954,10 +989,10 @@ my_SSL_set_fd(Port *port, int fd) SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - BIO_set_app_data(bio, port); + BIO_set_app_data(bio, layer); BIO_set_fd(bio, fd, BIO_NOCLOSE); - SSL_set_bio(port->ssl, bio, bio); + SSL_set_bio(ssl, bio, bio); ret = 1; err: return ret; diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c index a0f7084018..caee5b0556 100644 --- a/src/backend/libpq/be-secure.c +++ b/src/backend/libpq/be-secure.c @@ -125,219 +125,3 @@ secure_open_server(Port *port) return r; } - -/* - * Close secure session. - */ -void -secure_close(Port *port) -{ -#ifdef USE_SSL - if (port->ssl_in_use) - be_tls_close(port); -#endif -} - -/* - * Read data from a secure connection. - */ -ssize_t -secure_read(Port *port, void *ptr, size_t len) -{ - ssize_t n; - int waitfor; - - /* Deal with any already-pending interrupt condition. */ - ProcessClientReadInterrupt(false); - -retry: -#ifdef USE_SSL - waitfor = 0; - if (port->ssl_in_use) - { - n = be_tls_read(port, ptr, len, &waitfor); - } - else -#endif -#ifdef ENABLE_GSS - if (port->gss && port->gss->enc) - { - n = be_gssapi_read(port, ptr, len); - waitfor = WL_SOCKET_READABLE; - } - else -#endif - { - n = secure_raw_read(port, ptr, len); - waitfor = WL_SOCKET_READABLE; - } - - /* In blocking mode, wait until the socket is ready */ - if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) - { - WaitEvent event; - - Assert(waitfor); - - ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL); - - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, - WAIT_EVENT_CLIENT_READ); - - /* - * If the postmaster has died, it's not safe to continue running, - * because it is the postmaster's job to kill us if some other backend - * exits uncleanly. Moreover, we won't run very well in this state; - * helper processes like walwriter and the bgwriter will exit, so - * performance may be poor. Finally, if we don't exit, pg_ctl will be - * unable to restart the postmaster without manual intervention, so no - * new connections can be accepted. Exiting clears the deck for a - * postmaster restart. - * - * (Note that we only make this check when we would otherwise sleep on - * our latch. We might still continue running for a while if the - * postmaster is killed in mid-query, or even through multiple queries - * if we never have to wait for read. We don't want to burn too many - * cycles checking for this very rare condition, and this should cause - * us to exit quickly in most cases.) - */ - if (event.events & WL_POSTMASTER_DEATH) - ereport(FATAL, - (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("terminating connection due to unexpected postmaster exit"))); - - /* Handle interrupt. */ - if (event.events & WL_LATCH_SET) - { - ResetLatch(MyLatch); - ProcessClientReadInterrupt(true); - - /* - * We'll retry the read. Most likely it will return immediately - * because there's still no data available, and we'll wait for the - * socket to become ready again. - */ - } - goto retry; - } - - /* - * Process interrupts that happened during a successful (or non-blocking, - * or hard-failed) read. - */ - ProcessClientReadInterrupt(false); - - return n; -} - -ssize_t -secure_raw_read(Port *port, void *ptr, size_t len) -{ - ssize_t n; - - /* - * Try to read from the socket without blocking. If it succeeds we're - * done, otherwise we'll wait for the socket using the latch mechanism. - */ -#ifdef WIN32 - pgwin32_noblock = true; -#endif - n = recv(port->sock, ptr, len, 0); -#ifdef WIN32 - pgwin32_noblock = false; -#endif - - return n; -} - - -/* - * Write data to a secure connection. - */ -ssize_t -secure_write(Port *port, void *ptr, size_t len) -{ - ssize_t n; - int waitfor; - - /* Deal with any already-pending interrupt condition. */ - ProcessClientWriteInterrupt(false); - -retry: - waitfor = 0; -#ifdef USE_SSL - if (port->ssl_in_use) - { - n = be_tls_write(port, ptr, len, &waitfor); - } - else -#endif -#ifdef ENABLE_GSS - if (port->gss && port->gss->enc) - { - n = be_gssapi_write(port, ptr, len); - waitfor = WL_SOCKET_WRITEABLE; - } - else -#endif - { - n = secure_raw_write(port, ptr, len); - waitfor = WL_SOCKET_WRITEABLE; - } - - if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) - { - WaitEvent event; - - Assert(waitfor); - - ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL); - - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, - WAIT_EVENT_CLIENT_WRITE); - - /* See comments in secure_read. */ - if (event.events & WL_POSTMASTER_DEATH) - ereport(FATAL, - (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("terminating connection due to unexpected postmaster exit"))); - - /* Handle interrupt. */ - if (event.events & WL_LATCH_SET) - { - ResetLatch(MyLatch); - ProcessClientWriteInterrupt(true); - - /* - * We'll retry the write. Most likely it will return immediately - * because there's still no buffer space available, and we'll wait - * for the socket to become ready again. - */ - } - goto retry; - } - - /* - * Process interrupts that happened during a successful (or non-blocking, - * or hard-failed) write. - */ - ProcessClientWriteInterrupt(false); - - return n; -} - -ssize_t -secure_raw_write(Port *port, const void *ptr, size_t len) -{ - ssize_t n; - -#ifdef WIN32 - pgwin32_noblock = true; -#endif - n = send(port->sock, ptr, len, 0); -#ifdef WIN32 - pgwin32_noblock = false; -#endif - - return n; -} diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 0084a9bf13..030686cc3b 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -31,6 +31,7 @@ * setup/teardown: * StreamServerPort - Open postmaster's server port * StreamConnection - Create new connection with client + * StreamSetupIo - Configures IO stream on connection * StreamClose - Close a client/backend connection * TouchSocketFiles - Protect socket files against /tmp cleaners * pq_init - initialize libpq at backend startup @@ -74,12 +75,15 @@ #endif #include "common/ip.h" +#include "common/io_stream.h" #include "libpq/libpq.h" #include "miscadmin.h" #include "port/pg_bswap.h" #include "storage/ipc.h" +#include "tcop/tcopprot.h" #include "utils/guc_hooks.h" #include "utils/memutils.h" +#include "utils/wait_event.h" /* * Cope with the various platform-specific ways to spell TCP keepalive socket @@ -146,6 +150,15 @@ static int socket_putmessage(char msgtype, const char *s, size_t len); static void socket_putmessage_noblock(char msgtype, const char *s, size_t len); static int internal_putbytes(const char *s, size_t len); static int internal_flush(void); +static ssize_t socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int socket_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +static ssize_t io_read_with_wait(Port *port, void *ptr, size_t len); +static int io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_written); + +IoStreamProcessor socket_processor = { + .read = (io_stream_read_func) socket_read, + .write = (io_stream_write_func) socket_write +}; static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); static int Setup_AF_UNIX(const char *sock_path); @@ -277,7 +290,8 @@ socket_close(int code, Datum arg) * Cleanly shut down SSL layer. Nowhere else does a postmaster child * call this, so this is safe when interrupting BackendInitialize(). */ - secure_close(MyProcPort); + io_stream_destroy(MyProcPort->io_stream); + MyProcPort->io_stream = NULL; /* * Formerly we did an explicit close() here, but it seems better to @@ -817,6 +831,30 @@ StreamConnection(pgsocket server_fd, Port *port) return STATUS_OK; } +/* This must be called after the child process is launched or the data structures + * do not comre across correctly + */ +int +StreamSetupIo(Port *port) +{ + if (port->io_stream) + { + ereport(LOG, + (errmsg("%s() failed: io_stream already configured", "io_stream_create"))); + return STATUS_ERROR; + } + port->io_stream = io_stream_create(); + if (!port->io_stream) + { + ereport(LOG, + (errmsg("%s() failed: %m", "io_stream_create"))); + return STATUS_ERROR; + } + io_stream_add_layer(port->io_stream, &socket_processor, port); + + return STATUS_OK; +} + /* * StreamClose -- close a client/backend connection * @@ -905,6 +943,193 @@ socket_set_nonblocking(bool nonblocking) MyProcPort->noblock = nonblocking; } +static ssize_t +socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) +{ + ssize_t n; + + if (buffered_only) + return 0; + + /* + * Try to read from the socket without blocking. If it succeeds we're + * done, otherwise we'll wait for the socket using the latch mechanism. + */ +#ifdef WIN32 + pgwin32_noblock = true; +#endif + n = recv(port->sock, ptr, len, 0); +#ifdef WIN32 + pgwin32_noblock = false; +#endif + + return n; +} + +static int +socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size_t *bytes_written) +{ + ssize_t n; + +#ifdef WIN32 + pgwin32_noblock = true; +#endif + n = send(port->sock, ptr, len, 0); +#ifdef WIN32 + pgwin32_noblock = false; +#endif + + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } +} + +/* + * Read protocol-level data, processed through any intermediate streams like TLS + */ +static ssize_t +io_read_with_wait(Port *port, void *ptr, size_t len) +{ + ssize_t n; + + /* Deal with any already-pending interrupt condition. */ + ProcessClientReadInterrupt(false); + +retry: + port->waitfor = WL_SOCKET_READABLE; + n = io_stream_read(port->io_stream, ptr, len, false); + + /* In blocking mode, wait until the socket is ready */ + if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) + { + WaitEvent event; + + Assert(port->waitfor); + + ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL); + + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_READ); + + /* + * If the postmaster has died, it's not safe to continue running, + * because it is the postmaster's job to kill us if some other backend + * exits uncleanly. Moreover, we won't run very well in this state; + * helper processes like walwriter and the bgwriter will exit, so + * performance may be poor. Finally, if we don't exit, pg_ctl will be + * unable to restart the postmaster without manual intervention, so no + * new connections can be accepted. Exiting clears the deck for a + * postmaster restart. + * + * (Note that we only make this check when we would otherwise sleep on + * our latch. We might still continue running for a while if the + * postmaster is killed in mid-query, or even through multiple queries + * if we never have to wait for read. We don't want to burn too many + * cycles checking for this very rare condition, and this should cause + * us to exit quickly in most cases.) + */ + if (event.events & WL_POSTMASTER_DEATH) + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating connection due to unexpected postmaster exit"))); + + /* Handle interrupt. */ + if (event.events & WL_LATCH_SET) + { + ResetLatch(MyLatch); + ProcessClientReadInterrupt(true); + + /* + * We'll retry the read. Most likely it will return immediately + * because there's still no data available, and we'll wait for the + * socket to become ready again. + */ + } + goto retry; + } + + /* + * Process interrupts that happened during a successful (or non-blocking, + * or hard-failed) read. + */ + ProcessClientReadInterrupt(false); + + return n; +} + + +/* + * Write protocol-level data to be processed through any intermediate streams like TLS + */ +static int +io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_processed) +{ + int rc; + size_t count = 0; + + *bytes_processed = 0; + + /* Deal with any already-pending interrupt condition. */ + ProcessClientWriteInterrupt(false); + +retry: + port->waitfor = WL_SOCKET_WRITEABLE; + + /* + * on retry it is possible some of the input will have already been + * processed, so make sure we offset our retries + */ + rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count); + *bytes_processed += count; + + if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) + { + WaitEvent event; + + Assert(port->waitfor); + + ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL); + + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_WRITE); + + /* See comments in secure_read. */ + if (event.events & WL_POSTMASTER_DEATH) + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating connection due to unexpected postmaster exit"))); + + /* Handle interrupt. */ + if (event.events & WL_LATCH_SET) + { + ResetLatch(MyLatch); + ProcessClientWriteInterrupt(true); + + /* + * We'll retry the write. Most likely it will return immediately + * because there's still no buffer space available, and we'll wait + * for the socket to become ready again. + */ + } + goto retry; + } + + /* + * Process interrupts that happened during a successful (or non-blocking, + * or hard-failed) write. + */ + ProcessClientWriteInterrupt(false); + + return rc; +} + /* -------------------------------- * pq_recvbuf - load some bytes into the input buffer * @@ -938,8 +1163,8 @@ pq_recvbuf(void) errno = 0; - r = secure_read(MyProcPort, PqRecvBuffer + PqRecvLength, - PQ_RECV_BUFFER_SIZE - PqRecvLength); + r = io_read_with_wait(MyProcPort, PqRecvBuffer + PqRecvLength, + PQ_RECV_BUFFER_SIZE - PqRecvLength); if (r < 0) { @@ -1035,7 +1260,7 @@ pq_getbyte_if_available(unsigned char *c) errno = 0; - r = secure_read(MyProcPort, c, 1); + r = io_read_with_wait(MyProcPort, c, 1); if (r < 0) { /* @@ -1351,11 +1576,15 @@ internal_flush(void) while (bufptr < bufend) { - int r; + int rc; + size_t bytes_sent; + size_t available = bufend - bufptr; - r = secure_write(MyProcPort, bufptr, bufend - bufptr); + rc = io_write_with_wait(MyProcPort, bufptr, available, &bytes_sent); + bufptr += bytes_sent; + PqSendStart += bytes_sent; - if (r <= 0) + if (rc < 0 || (bytes_sent == 0 && available)) { if (errno == EINTR) continue; /* Ok if we were interrupted */ @@ -1400,8 +1629,6 @@ internal_flush(void) } last_reported_send_errno = 0; /* reset after any successful send */ - bufptr += r; - PqSendStart += r; } PqSendStart = PqSendPointer = 0; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 651b85ea74..14122de017 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4193,6 +4193,11 @@ BackendInitialize(Port *port) /* Save port etc. for ps status */ MyProcPort = port; + if (StreamSetupIo(port)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to configure backend I/O"))); + /* Tell fd.c about the long-lived FD associated with the port */ ReserveExternalFD(); diff --git a/src/common/Makefile b/src/common/Makefile index ce4535d7fe..a5cd11c6df 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -60,6 +60,7 @@ OBJS_COMMON = \ file_perm.o \ file_utils.o \ hashfn.o \ + io_stream.o \ ip.o \ jsonapi.o \ keywords.o \ diff --git a/src/common/io_stream.c b/src/common/io_stream.c new file mode 100644 index 0000000000..b15aca326d --- /dev/null +++ b/src/common/io_stream.c @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * io_stream.c + * functions related to managing layers of streaming IO. + * In general the base layer will work with raw sockets, and then + * additional layers will add features such as encryption and + * compression. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/io_stream.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include <common/io_stream.h> + +#ifndef FRONTEND +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define FREE(size) free(size) +#endif + +struct IoStreamLayer +{ + IoStreamProcessor *processor; + void *context; + IoStreamLayer *next; +}; + +struct IoStream +{ + IoStreamLayer *layer; +}; + +IoStream * +io_stream_create(void) +{ + IoStream *ret = ALLOC(sizeof(IoStream)); + + ret->layer = NULL; + return ret; +} + +void +io_stream_destroy(IoStream * arg) +{ + IoStreamLayer *layer; + + if (arg == NULL) + return; + + layer = arg->layer; + while (layer != NULL) + { + IoStreamLayer *next = layer->next; + + if (layer->processor->destroy != NULL) + layer->processor->destroy(layer->context); + FREE(layer); + layer = next; + } + FREE(arg); +} + +IoStreamLayer * +io_stream_add_layer(IoStream * stream, IoStreamProcessor * processor, void *context) +{ + IoStreamLayer *layer = ALLOC(sizeof(IoStreamLayer)); + + layer->processor = processor; + layer->context = context; + layer->next = stream->layer; + stream->layer = layer; + return layer; +} + +ssize_t +io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only) +{ + if (stream->layer == NULL) + return -1; + + return stream->layer->processor->read(stream->layer, stream->layer->context, data, size, buffered_only); +} + +int +io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written) +{ + if (stream->layer == NULL) + return -1; + + return stream->layer->processor->write(stream->layer, stream->layer->context, data, size, bytes_written); +} + +bool +io_stream_buffered_read_data(IoStream * stream) +{ + IoStreamLayer *layer; + + for (layer = stream->layer; layer != NULL; layer = layer->next) + { + if (layer->processor->buffered_read_data != NULL && layer->processor->buffered_read_data(layer->context)) + return true; + } + return false; +} + +bool +io_stream_buffered_write_data(IoStream * stream) +{ + IoStreamLayer *layer; + + for (layer = stream->layer; layer != NULL; layer = layer->next) + { + if (layer->processor->buffered_write_data != NULL && layer->processor->buffered_write_data(layer->context)) + return true; + } + return false; +} + +ssize_t +io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only) +{ + if (layer->next == NULL) + return -1; + + return layer->next->processor->read(layer->next, layer->next->context, data, size, buffered_only); +} + +int +io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written) +{ + if (layer->next == NULL) + return -1; + + return layer->next->processor->write(layer->next, layer->next->context, data, size, bytes_written); +} diff --git a/src/common/meson.build b/src/common/meson.build index 8be145c0fb..3e819108a1 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -12,6 +12,7 @@ common_sources = files( 'file_perm.c', 'file_utils.c', 'hashfn.c', + 'io_stream.c', 'ip.c', 'jsonapi.c', 'keywords.c', diff --git a/src/include/common/io_stream.h b/src/include/common/io_stream.h new file mode 100644 index 0000000000..9af88aa9ea --- /dev/null +++ b/src/include/common/io_stream.h @@ -0,0 +1,131 @@ +/*------------------------------------------------------------------------- + * + * io_stream.c + * defintions for managing layers of streaming IO. + * In general the base layer will work with raw sockets, and then + * additional layers will add features such as encryption and + * compression. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/common/io_stream.h + * + *------------------------------------------------------------------------- + */ +#ifndef IO_STREAM_H +#define IO_STREAM_H + +/* Opaque structs should only be interacted with through corresponding functions*/ +typedef struct IoStreamLayer IoStreamLayer; +typedef struct IoStream IoStream; + +typedef ssize_t (*io_stream_read_func) (IoStreamLayer * self, void *context, void *data, size_t size, bool buffered_only); +typedef int (*io_stream_write_func) (IoStreamLayer * self, void *context, void const *data, size_t size, size_t *bytes_written); +typedef bool (*io_stream_predicate) (void *context); +typedef void (*io_stream_destroy_func) (void *context); + +typedef struct IoStreamProcessor +{ + /* + * Required Should call io_stream_next_read with self either directly or + * indirectly to recieve bytes from the underlying layer of the stream + */ + io_stream_read_func read; + + /* + * Required Should call io_stream_next_write with self either directly or + * indirectly to write bytes to the underlying layer of the stream + */ + io_stream_write_func write; + + /* + * Optional Return true if this layer is holding buffered readable data + */ + io_stream_predicate buffered_read_data; + + /* + * Optional Return true if this layer is holding buffered writable data + */ + io_stream_predicate buffered_write_data; + + /* + * Optional will be called as part of io_stream_destroy when cleaning up + * the stream + */ + io_stream_destroy_func destroy; +} IoStreamProcessor; + +/* + * Allocate a new IoStream and return the address to the caller. IoStreams should always be destroyed with + * the corresponding io_stream_destroy function + */ +extern IoStream * io_stream_create(void); +/* + * Adds new processors to the IO stream + * + * processorDescription contains collection of function pointers for this layer + * context (optional) pointer with context to be used in reader/writer + * + * Returns the newly created processor + * */ +extern IoStreamLayer * io_stream_add_layer(IoStream * stream, IoStreamProcessor * processorDescription, void *context); +/* + * Destroy an IoStream, freeing all associated memory + */ +extern void io_stream_destroy(IoStream * stream); + +/* + * Read data from the stream + * + * Reads at most size bytes into the buffer pointed to by data, and returns + * the number of bytes read. If buffered_only is true, then only data that + * was stored in an in-process buffer will be returned and this function will + * not block. In that case, a return value of 0 simply means there was no + * buffered data available and does not mean the stream has reached EOF. + */ +extern ssize_t io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only); + +/* + * Write data to the stream + * + * Writes at most size bytes from the buffer pointed to by data, and returns + * true on success, false on failure, with the specific error in errno. The + * count of bytes written from data will be stored in bytes_written and may + * be non-zero even on failure + */ +extern int io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written); + +/* + * Check if there is data buffered in memory waiting to be read (e.g. if + * compression is in use and more uncompressed data was read than fit into + * the provided buffer) + */ +extern bool io_stream_buffered_read_data(IoStream * stream); + +/* + * Check if there is data buffered in memory waiting to be written to the underlying backing store + */ +extern bool io_stream_buffered_write_data(IoStream * stream); + +/* + * Read data from the next layer of the stream + * (to be used by io_stream_read_func) + * + * Reads at most size bytes into the buffer pointed to by data, and returns + * the number of bytes read + */ +extern ssize_t io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only); + +/* + * Write data to the next layer of the stream + * (to be used by io_stream_write_func) + * + * Writes at most size bytes from the buffer pointed to by data, and returns + * true on success, false on failure, with the specific error in errno. The + * count of bytes written from data will be stored in bytes_written and may + * be non-zero even on failure + */ +extern int io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written); + +#endif /* //IO_STREAM_H */ diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index c57ed12fb6..87ba7f5ea0 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -53,6 +53,7 @@ typedef struct #endif #endif /* ENABLE_SSPI */ +#include "common/io_stream.h" #include "datatype/timestamp.h" #include "libpq/hba.h" #include "libpq/pqcomm.h" @@ -145,8 +146,11 @@ typedef struct ClientConnectionInfo typedef struct Port { + IoStream *io_stream; pgsocket sock; /* File descriptor */ bool noblock; /* is the socket in non-blocking mode? */ + int waitfor; /* Events to wait on the socket for after + * attempted read/write */ ProtocolVersion proto; /* FE/BE protocol version */ SockAddr laddr; /* local addr (postmaster) */ SockAddr raddr; /* remote addr (client) */ @@ -274,21 +278,6 @@ extern void be_tls_destroy(void); */ extern int be_tls_open_server(Port *port); -/* - * Close SSL connection. - */ -extern void be_tls_close(Port *port); - -/* - * Read data from a secure connection. - */ -extern ssize_t be_tls_read(Port *port, void *ptr, size_t len, int *waitfor); - -/* - * Write data to a secure connection. - */ -extern ssize_t be_tls_write(Port *port, void *ptr, size_t len, int *waitfor); - /* * Return information about the SSL connection. */ @@ -324,10 +313,6 @@ extern bool be_gssapi_get_auth(Port *port); extern bool be_gssapi_get_enc(Port *port); extern const char *be_gssapi_get_princ(Port *port); extern bool be_gssapi_get_delegation(Port *port); - -/* Read and write to a GSSAPI-encrypted connection. */ -extern ssize_t be_gssapi_read(Port *port, void *ptr, size_t len); -extern ssize_t be_gssapi_write(Port *port, void *ptr, size_t len); #endif /* ENABLE_GSS */ extern PGDLLIMPORT ProtocolVersion FrontendProtocol; diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h index a6104d8cd0..3612280146 100644 --- a/src/include/libpq/libpq.h +++ b/src/include/libpq/libpq.h @@ -68,6 +68,7 @@ extern int StreamServerPort(int family, const char *hostName, unsigned short portNumber, const char *unixSocketDir, pgsocket ListenSocket[], int *NumListenSockets, int MaxListen); extern int StreamConnection(pgsocket server_fd, Port *port); +extern int StreamSetupIo(Port *port); extern void StreamClose(pgsocket sock); extern void TouchSocketFiles(void); extern void RemoveSocketFiles(void); @@ -104,11 +105,6 @@ extern int secure_initialize(bool isServerStart); extern bool secure_loaded_verify_locations(void); extern void secure_destroy(void); extern int secure_open_server(Port *port); -extern void secure_close(Port *port); -extern ssize_t secure_read(Port *port, void *ptr, size_t len); -extern ssize_t secure_write(Port *port, void *ptr, size_t len); -extern ssize_t secure_raw_read(Port *port, void *ptr, size_t len); -extern ssize_t secure_raw_write(Port *port, const void *ptr, size_t len); /* * prototypes for functions in be-secure-gssapi.c diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index bf83a9b569..a0f12e62af 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -136,6 +136,55 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options, #define DefaultGSSMode "disable" #endif +/* + * Macros to handle disabling and then restoring the state of SIGPIPE handling. + * On Windows, these are all no-ops since there's no SIGPIPEs. + */ + +#ifndef WIN32 + +#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag) + +struct sigpipe_info +{ + sigset_t oldsigmask; + bool sigpipe_pending; + bool got_epipe; +}; + +#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo + +#define DISABLE_SIGPIPE(conn, spinfo, failaction) \ +do { \ +(spinfo).got_epipe = false; \ +if (!SIGPIPE_MASKED(conn)) \ +{ \ +if (pq_block_sigpipe(&(spinfo).oldsigmask, \ +&(spinfo).sigpipe_pending) < 0) \ +failaction; \ +} \ +} while (0) + +#define REMEMBER_EPIPE(spinfo, cond) \ +do { \ +if (cond) \ +(spinfo).got_epipe = true; \ +} while (0) + +#define RESTORE_SIGPIPE(conn, spinfo) \ +do { \ +if (!SIGPIPE_MASKED(conn)) \ +pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \ +(spinfo).got_epipe); \ +} while (0) +#else /* WIN32 */ + +#define DECLARE_SIGPIPE_INFO(spinfo) +#define DISABLE_SIGPIPE(conn, spinfo, failaction) +#define REMEMBER_EPIPE(spinfo, cond) +#define RESTORE_SIGPIPE(conn, spinfo) +#endif /* WIN32 */ + /* ---------- * Definition of the conninfo parameters and their fallback resources. * @@ -445,6 +494,12 @@ static bool sslVerifyProtocolVersion(const char *version); static bool sslVerifyProtocolRange(const char *min, const char *max); static bool parse_int_param(const char *value, int *result, PGconn *conn, const char *context); +static ssize_t socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written); +IoStreamProcessor socket_processor = { + .read = (io_stream_read_func) socket_read, + .write = (io_stream_write_func) socket_write +}; /* global variable because fe-auth.c needs to access it */ @@ -466,9 +521,8 @@ pgthreadlock_t pg_g_threadlock = default_threadlock; void pqDropConnection(PGconn *conn, bool flushInput) { - /* Drop any SSL state */ - pqsecure_close(conn); - + io_stream_destroy(conn->io_stream); + conn->io_stream = NULL; /* Close the socket itself */ if (conn->sock != PGINVALID_SOCKET) closesocket(conn->sock); @@ -2879,6 +2933,9 @@ keep_going: /* We will come back to here until there is sock_type |= SOCK_NONBLOCK; #endif conn->sock = socket(addr_cur->family, sock_type, 0); + conn->io_stream = io_stream_create(); + io_stream_add_layer(conn->io_stream, &socket_processor, conn); + if (conn->sock == PGINVALID_SOCKET) { int errorno = SOCK_ERRNO; @@ -7829,3 +7886,283 @@ PQregisterThreadLock(pgthreadlock_t newhandler) return prev; } + +static ssize_t +socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) +{ + ssize_t n; + int result_errno = 0; + char sebuf[PG_STRERROR_R_BUFLEN]; + + SOCK_ERRNO_SET(0); + + if (buffered_only) + return 0; + + n = recv(conn->sock, ptr, len, 0); + + if (n < 0) + { + result_errno = SOCK_ERRNO; + + /* Set error message if appropriate */ + switch (result_errno) + { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) + case EWOULDBLOCK: +#endif + case EINTR: + /* no error message, caller is expected to retry */ + break; + + case EPIPE: + case ECONNRESET: + libpq_append_conn_error(conn, "server closed the connection unexpectedly\n" + "\tThis probably means the server terminated abnormally\n" + "\tbefore or while processing the request."); + break; + + case 0: + /* If errno didn't get set, treat it as regular EOF */ + n = 0; + break; + + default: + libpq_append_conn_error(conn, "could not receive data from server: %s", + SOCK_STRERROR(result_errno, + sebuf, sizeof(sebuf))); + break; + } + } + + /* ensure we return the intended errno to caller */ + SOCK_ERRNO_SET(result_errno); + + return n; +} + +/* + * Socket-level implementation of data writing. + * + * This is used directly for an unencrypted connection. For encrypted + * connections, this is wrapped by higher layers through IO stream + * + * This function reports failure (i.e., returns a negative result) only + * for retryable errors such as EINTR. Looping for such cases is to be + * handled at some outer level, maybe all the way up to the application. + * For hard failures, we set conn->write_failed and store an error message + * in conn->write_err_msg, but then claim to have written the data anyway. + * This is because we don't want to report write failures so long as there + * is a possibility of reading from the server and getting an error message + * that could explain why the connection dropped. Many TCP stacks have + * race conditions such that a write failure may or may not be reported + * before all incoming data has been read. + * + * Note that this error behavior happens below the SSL management level when + * we are using SSL. That's because at least some versions of OpenSSL are + * too quick to report a write failure when there's still a possibility to + * get a more useful error from the server. + */ +static int +socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written) +{ + ssize_t n; + int flags = 0; + int result_errno = 0; + char msgbuf[1024]; + char sebuf[PG_STRERROR_R_BUFLEN]; + + DECLARE_SIGPIPE_INFO(spinfo); + + /* + * If we already had a write failure, we will never again try to send data + * on that connection. Even if the kernel would let us, we've probably + * lost message boundary sync with the server. conn->write_failed + * therefore persists until the connection is reset, and we just discard + * all data presented to be written. + */ + if (conn->write_failed) + return len; + +#ifdef MSG_NOSIGNAL + if (conn->sigpipe_flag) + flags |= MSG_NOSIGNAL; + +retry_masked: +#endif /* MSG_NOSIGNAL */ + + DISABLE_SIGPIPE(conn, spinfo, return -1); + + n = send(conn->sock, ptr, len, flags); + + if (n < 0) + { + *bytes_written = 0; + result_errno = SOCK_ERRNO; + + /* + * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available + * on this machine. So, clear sigpipe_flag so we don't try the flag + * again, and retry the send(). + */ +#ifdef MSG_NOSIGNAL + if (flags != 0 && result_errno == EINVAL) + { + conn->sigpipe_flag = false; + flags = 0; + goto retry_masked; + } +#endif /* MSG_NOSIGNAL */ + + /* Set error message if appropriate */ + switch (result_errno) + { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) + case EWOULDBLOCK: +#endif + case EINTR: + /* no error message, caller is expected to retry */ + break; + + case EPIPE: + /* Set flag for EPIPE */ + REMEMBER_EPIPE(spinfo, true); + + /* FALL THRU */ + + case ECONNRESET: + conn->write_failed = true; + /* Store error message in conn->write_err_msg, if possible */ + /* (strdup failure is OK, we'll cope later) */ + snprintf(msgbuf, sizeof(msgbuf), + libpq_gettext("server closed the connection unexpectedly\n" + "\tThis probably means the server terminated abnormally\n" + "\tbefore or while processing the request.")); + /* keep newline out of translated string */ + strlcat(msgbuf, "\n", sizeof(msgbuf)); + conn->write_err_msg = strdup(msgbuf); + /* Now claim the write succeeded */ + n = len; + break; + + default: + conn->write_failed = true; + /* Store error message in conn->write_err_msg, if possible */ + /* (strdup failure is OK, we'll cope later) */ + snprintf(msgbuf, sizeof(msgbuf), + libpq_gettext("could not send data to server: %s"), + SOCK_STRERROR(result_errno, + sebuf, sizeof(sebuf))); + /* keep newline out of translated string */ + strlcat(msgbuf, "\n", sizeof(msgbuf)); + conn->write_err_msg = strdup(msgbuf); + /* Now claim the write succeeded */ + n = len; + break; + } + } + else + { + *bytes_written = n; + n = 0; + } + + RESTORE_SIGPIPE(conn, spinfo); + + /* ensure we return the intended errno to caller */ + SOCK_ERRNO_SET(result_errno); + + return n; +} + +#if !defined(WIN32) + +/* + * Block SIGPIPE for this thread. This prevents send()/write() from exiting + * the application. + */ +int +pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending) +{ + sigset_t sigpipe_sigset; + sigset_t sigset; + + sigemptyset(&sigpipe_sigset); + sigaddset(&sigpipe_sigset, SIGPIPE); + + /* Block SIGPIPE and save previous mask for later reset */ + SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset)); + if (SOCK_ERRNO) + return -1; + + /* We can have a pending SIGPIPE only if it was blocked before */ + if (sigismember(osigset, SIGPIPE)) + { + /* Is there a pending SIGPIPE? */ + if (sigpending(&sigset) != 0) + return -1; + + if (sigismember(&sigset, SIGPIPE)) + *sigpipe_pending = true; + else + *sigpipe_pending = false; + } + else + *sigpipe_pending = false; + + return 0; +} + +/* + * Discard any pending SIGPIPE and reset the signal mask. + * + * Note: we are effectively assuming here that the C library doesn't queue + * up multiple SIGPIPE events. If it did, then we'd accidentally leave + * ours in the queue when an event was already pending and we got another. + * As long as it doesn't queue multiple events, we're OK because the caller + * can't tell the difference. + * + * The caller should say got_epipe = false if it is certain that it + * didn't get an EPIPE error; in that case we'll skip the clear operation + * and things are definitely OK, queuing or no. If it got one or might have + * gotten one, pass got_epipe = true. + * + * We do not want this to change errno, since if it did that could lose + * the error code from a preceding send(). We essentially assume that if + * we were able to do pq_block_sigpipe(), this can't fail. + */ +void +pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe) +{ + int save_errno = SOCK_ERRNO; + int signo; + sigset_t sigset; + + /* Clear SIGPIPE only if none was pending */ + if (got_epipe && !sigpipe_pending) + { + if (sigpending(&sigset) == 0 && + sigismember(&sigset, SIGPIPE)) + { + sigset_t sigpipe_sigset; + + sigemptyset(&sigpipe_sigset); + sigaddset(&sigpipe_sigset, SIGPIPE); + + sigwait(&sigpipe_sigset, &signo); + } + } + + /* Restore saved block mask */ + pthread_sigmask(SIG_SETMASK, osigset, NULL); + + SOCK_ERRNO_SET(save_errno); +} + +#endif /* !WIN32 */ diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index 660cdec93c..6772f2876d 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -614,8 +614,8 @@ pqReadData(PGconn *conn) /* OK, try to read some data */ retry3: - nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, - conn->inBufSize - conn->inEnd); + nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, + conn->inBufSize - conn->inEnd, false); if (nread < 0) { switch (SOCK_ERRNO) @@ -709,8 +709,8 @@ retry3: * arrived. */ retry4: - nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, - conn->inBufSize - conn->inEnd); + nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, + conn->inBufSize - conn->inEnd, false); if (nread < 0) { switch (SOCK_ERRNO) @@ -824,10 +824,11 @@ pqSendSome(PGconn *conn, int len) /* while there's still data to send */ while (len > 0) { - int sent; + size_t sent; + int rc; #ifndef WIN32 - sent = pqsecure_write(conn, ptr, len); + rc = io_stream_write(conn->io_stream, ptr, len, &sent); #else /* @@ -835,10 +836,13 @@ pqSendSome(PGconn *conn, int len) * failure-point appears to be different in different versions of * Windows, but 64k should always be safe. */ - sent = pqsecure_write(conn, ptr, Min(len, 65536)); + rc = io_stream_write(conn->io_stream, ptr, Min(len, 65536), &sent); #endif + ptr += sent; + len -= sent; + remaining -= sent; - if (sent < 0) + if (rc < 0) { /* Anything except EAGAIN/EWOULDBLOCK/EINTR is trouble */ switch (SOCK_ERRNO) @@ -878,12 +882,6 @@ pqSendSome(PGconn *conn, int len) return -1; } } - else - { - ptr += sent; - len -= sent; - remaining -= sent; - } if (len > 0) { @@ -1048,14 +1046,11 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) return -1; } -#ifdef USE_SSL - /* Check for SSL library buffering read bytes */ - if (forRead && conn->ssl_in_use && pgtls_read_pending(conn)) + if (forRead && io_stream_buffered_read_data(conn->io_stream)) { /* short-circuit the select */ return 1; } -#endif /* We will retry as long as we get EINTR */ do diff --git a/src/interfaces/libpq/fe-secure-gssapi.c b/src/interfaces/libpq/fe-secure-gssapi.c index ea8b0020d2..85cec6ced3 100644 --- a/src/interfaces/libpq/fe-secure-gssapi.c +++ b/src/interfaces/libpq/fe-secure-gssapi.c @@ -69,6 +69,13 @@ #define PqGSSResultNext (conn->gss_ResultNext) #define PqGSSMaxPktSize (conn->gss_MaxPktSize) +static ssize_t pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written); + +IoStreamProcessor pg_GSS_processor = { + .read = (io_stream_read_func) pg_GSS_read, + .write = (io_stream_write_func) pg_GSS_write +}; /* * Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection. @@ -82,8 +89,8 @@ * For retryable errors, caller should call again (passing the same or more * data) once the socket is ready. */ -ssize_t -pg_GSS_write(PGconn *conn, const void *ptr, size_t len) +static int +pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written) { OM_uint32 major, minor; @@ -94,6 +101,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) size_t bytes_encrypted; gss_ctx_id_t gctx = conn->gctx; + *bytes_written = 0; + /* * When we get a retryable failure, we must not tell the caller we have * successfully transmitted everything, else it won't retry. For @@ -124,7 +133,7 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) /* * Loop through encrypting data and sending it out until it's all done or - * pqsecure_raw_write() complains (which would likely mean that the socket + * io_stream_next_write() complains (which would likely mean that the socket * is non-blocking and the requested send() would block, or there was some * kind of actual error). */ @@ -141,20 +150,21 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) */ if (PqGSSSendLength) { - ssize_t retval; + int retval; + size_t count; ssize_t amount = PqGSSSendLength - PqGSSSendNext; - retval = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount); - if (retval <= 0) + retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count); + if (retval < 0 || count == 0) return retval; /* * Check if this was a partial write, and if so, move forward that * far in our buffer and try again. */ - if (retval < amount) + if (count < amount) { - PqGSSSendNext += retval; + PqGSSSendNext += count; continue; } @@ -235,7 +245,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) /* We're reporting all the data as sent, so reset PqGSSSendConsumed. */ PqGSSSendConsumed = 0; - ret = bytes_encrypted; + ret = 0; + *bytes_written = bytes_encrypted; cleanup: /* Release GSSAPI buffer storage, if we didn't already */ @@ -255,8 +266,8 @@ cleanup: * error, a message is added to conn->errorMessage. For retryable errors, * caller should call again once the socket is ready. */ -ssize_t -pg_GSS_read(PGconn *conn, void *ptr, size_t len) +static ssize_t +pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) { OM_uint32 major, minor; @@ -266,6 +277,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) size_t bytes_returned = 0; gss_ctx_id_t gctx = conn->gctx; + if (buffered_only) + return 0; + /* * The plan here is to read one incoming encrypted packet into * PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out @@ -322,10 +336,10 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) /* Collect the length if we haven't already */ if (PqGSSRecvLength < sizeof(uint32)) { - ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength, - sizeof(uint32) - PqGSSRecvLength); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + sizeof(uint32) - PqGSSRecvLength, false); - /* If ret <= 0, pqsecure_raw_read already set the correct errno */ + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -355,9 +369,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) * Read as much of the packet as we are able to on this call into * wherever we left off from the last time we were called. */ - ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength, - input.length - (PqGSSRecvLength - sizeof(uint32))); - /* If ret <= 0, pqsecure_raw_read already set the correct errno */ + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + input.length - (PqGSSRecvLength - sizeof(uint32)), false); + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -418,16 +432,17 @@ cleanup: } /* - * Simple wrapper for reading from pqsecure_raw_read. + * Simple wrapper for reading from io_stream_read. Only used during connecition setup + * before GSS is added to the io_stream. * - * This takes the same arguments as pqsecure_raw_read, plus an output parameter + * This takes the same arguments as io_stream_read, plus an output parameter * to return the number of bytes read. This handles if blocking would occur and * if we detect EOF on the connection. */ static PostgresPollingStatusType gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret) { - *ret = pqsecure_raw_read(conn, recv_buffer, length); + *ret = io_stream_read(conn->io_stream, recv_buffer, length, false); if (*ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -447,7 +462,7 @@ gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret) if (!result) return PGRES_POLLING_READING; - *ret = pqsecure_raw_read(conn, recv_buffer, length); + *ret = io_stream_read(conn->io_stream, recv_buffer, length, false); if (*ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -506,8 +521,9 @@ pqsecure_open_gss(PGconn *conn) if (PqGSSSendLength) { ssize_t amount = PqGSSSendLength - PqGSSSendNext; + size_t count; - ret = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount); + ret = io_stream_write(conn->io_stream, PqGSSSendBuffer + PqGSSSendNext, amount, &count); if (ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -516,7 +532,7 @@ pqsecure_open_gss(PGconn *conn) return PGRES_POLLING_FAILED; } - if (ret < amount) + if (count < amount) { PqGSSSendNext += ret; return PGRES_POLLING_WRITING; @@ -662,6 +678,8 @@ pqsecure_open_gss(PGconn *conn) */ conn->gssenc = true; conn->gssapi_used = true; + io_stream_add_layer(conn->io_stream, &pg_GSS_processor, conn); + /* Clean up */ gss_release_cred(&minor, &conn->gcred); diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 2b221e7d15..b9706cb151 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -81,8 +81,17 @@ static int PQssl_passwd_cb(char *buf, int size, int rwflag, void *userdata); static int my_sock_read(BIO *h, char *buf, int size); static int my_sock_write(BIO *h, const char *buf, int size); static BIO_METHOD *my_BIO_s_socket(void); -static int my_SSL_set_fd(PGconn *conn, int fd); - +static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer); +static ssize_t pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int pgtls_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written); +static bool pgtls_read_pending(PGconn *conn); +static void pgtls_close(PGconn *conn); +IoStreamProcessor pgtls_processor = { + .read = (io_stream_read_func) pgtls_read, + .write = (io_stream_write_func) pgtls_write, + .buffered_read_data = (io_stream_predicate) pgtls_read_pending, + .destroy = (io_stream_destroy_func) pgtls_close +}; static bool pq_init_ssl_lib = true; static bool pq_init_crypto_lib = true; @@ -141,8 +150,8 @@ pgtls_open_client(PGconn *conn) return open_client_SSL(conn); } -ssize_t -pgtls_read(PGconn *conn, void *ptr, size_t len) +static ssize_t +pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) { ssize_t n; int result_errno = 0; @@ -150,8 +159,20 @@ pgtls_read(PGconn *conn, void *ptr, size_t len) int err; unsigned long ecode; + if (buffered_only) + { + /* + * SSL_pending bytes are guaranteed to be available and readable + * without blocking + */ + len = Min(len, SSL_pending(conn->ssl)); + if (len == 0) + return 0; + } + rloop: + /* * Prepare to call SSL_get_error() by clearing thread's OpenSSL error * queue. In general, the current thread's error queue must be empty @@ -257,14 +278,14 @@ rloop: return n; } -bool +static bool pgtls_read_pending(PGconn *conn) { return SSL_pending(conn->ssl) > 0; } -ssize_t -pgtls_write(PGconn *conn, const void *ptr, size_t len) +static int +pgtls_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written) { ssize_t n; int result_errno = 0; @@ -360,7 +381,17 @@ pgtls_write(PGconn *conn, const void *ptr, size_t len) /* ensure we return the intended errno to caller */ SOCK_ERRNO_SET(result_errno); - return n; + + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } } char * @@ -1211,7 +1242,8 @@ initialize_SSL(PGconn *conn) */ if (!(conn->ssl = SSL_new(SSL_context)) || !SSL_set_app_data(conn->ssl, conn) || - !my_SSL_set_fd(conn, conn->sock)) + !my_SSL_set_fd(conn->ssl, conn->sock, + io_stream_add_layer(conn->io_stream, &pgtls_processor, conn))) { char *err = SSLerrmessage(ERR_get_error()); @@ -1613,7 +1645,7 @@ open_client_SSL(PGconn *conn) return PGRES_POLLING_OK; } -void +static void pgtls_close(PGconn *conn) { bool destroy_needed = false; @@ -1815,7 +1847,7 @@ PQsslAttribute(PGconn *conn, const char *attribute_name) /* * Private substitute BIO: this does the sending and receiving using - * pqsecure_raw_write() and pqsecure_raw_read() instead, to allow those + * io_stream_next_write() and io_stream_next_read() instead, to allow those * functions to disable SIGPIPE and give better error messages on I/O errors. * * These functions are closely modelled on the standard socket BIO in OpenSSL; @@ -1830,7 +1862,7 @@ my_sock_read(BIO *h, char *buf, int size) { int res; - res = pqsecure_raw_read((PGconn *) BIO_get_app_data(h), buf, size); + res = io_stream_next_read(BIO_get_app_data(h), buf, size, false); BIO_clear_retry_flags(h); if (res < 0) { @@ -1859,8 +1891,9 @@ static int my_sock_write(BIO *h, const char *buf, int size) { int res; + size_t count; - res = pqsecure_raw_write((PGconn *) BIO_get_app_data(h), buf, size); + res = io_stream_next_write(BIO_get_app_data(h), buf, size, &count); BIO_clear_retry_flags(h); if (res < 0) { @@ -1882,7 +1915,7 @@ my_sock_write(BIO *h, const char *buf, int size) } } - return res; + return res == 0 ? count : res; } static BIO_METHOD * @@ -1952,7 +1985,7 @@ err: /* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */ static int -my_SSL_set_fd(PGconn *conn, int fd) +my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer) { int ret = 0; BIO *bio; @@ -1965,15 +1998,16 @@ my_SSL_set_fd(PGconn *conn, int fd) goto err; } bio = BIO_new(bio_method); + if (bio == NULL) { SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - BIO_set_app_data(bio, conn); + BIO_set_app_data(bio, layer); - SSL_set_bio(conn->ssl, bio, bio); BIO_set_fd(bio, fd, BIO_NOCLOSE); + SSL_set_bio(ssl, bio, bio); ret = 1; err: return ret; diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index b2430362a9..4b34de0220 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -45,55 +45,6 @@ #include "libpq-fe.h" #include "libpq-int.h" -/* - * Macros to handle disabling and then restoring the state of SIGPIPE handling. - * On Windows, these are all no-ops since there's no SIGPIPEs. - */ - -#ifndef WIN32 - -#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag) - -struct sigpipe_info -{ - sigset_t oldsigmask; - bool sigpipe_pending; - bool got_epipe; -}; - -#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo - -#define DISABLE_SIGPIPE(conn, spinfo, failaction) \ - do { \ - (spinfo).got_epipe = false; \ - if (!SIGPIPE_MASKED(conn)) \ - { \ - if (pq_block_sigpipe(&(spinfo).oldsigmask, \ - &(spinfo).sigpipe_pending) < 0) \ - failaction; \ - } \ - } while (0) - -#define REMEMBER_EPIPE(spinfo, cond) \ - do { \ - if (cond) \ - (spinfo).got_epipe = true; \ - } while (0) - -#define RESTORE_SIGPIPE(conn, spinfo) \ - do { \ - if (!SIGPIPE_MASKED(conn)) \ - pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \ - (spinfo).got_epipe); \ - } while (0) -#else /* WIN32 */ - -#define DECLARE_SIGPIPE_INFO(spinfo) -#define DISABLE_SIGPIPE(conn, spinfo, failaction) -#define REMEMBER_EPIPE(spinfo, cond) -#define RESTORE_SIGPIPE(conn, spinfo) -#endif /* WIN32 */ - /* ------------------------------------------------------------ */ /* Procedures common to all secure sessions */ /* ------------------------------------------------------------ */ @@ -160,281 +111,6 @@ pqsecure_open_client(PGconn *conn) #endif } -/* - * Close secure session. - */ -void -pqsecure_close(PGconn *conn) -{ -#ifdef USE_SSL - pgtls_close(conn); -#endif -} - -/* - * Read data from a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -ssize_t -pqsecure_read(PGconn *conn, void *ptr, size_t len) -{ - ssize_t n; - -#ifdef USE_SSL - if (conn->ssl_in_use) - { - n = pgtls_read(conn, ptr, len); - } - else -#endif -#ifdef ENABLE_GSS - if (conn->gssenc) - { - n = pg_GSS_read(conn, ptr, len); - } - else -#endif - { - n = pqsecure_raw_read(conn, ptr, len); - } - - return n; -} - -ssize_t -pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) -{ - ssize_t n; - int result_errno = 0; - char sebuf[PG_STRERROR_R_BUFLEN]; - - SOCK_ERRNO_SET(0); - - n = recv(conn->sock, ptr, len, 0); - - if (n < 0) - { - result_errno = SOCK_ERRNO; - - /* Set error message if appropriate */ - switch (result_errno) - { -#ifdef EAGAIN - case EAGAIN: -#endif -#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - case EWOULDBLOCK: -#endif - case EINTR: - /* no error message, caller is expected to retry */ - break; - - case EPIPE: - case ECONNRESET: - libpq_append_conn_error(conn, "server closed the connection unexpectedly\n" - "\tThis probably means the server terminated abnormally\n" - "\tbefore or while processing the request."); - break; - - case 0: - /* If errno didn't get set, treat it as regular EOF */ - n = 0; - break; - - default: - libpq_append_conn_error(conn, "could not receive data from server: %s", - SOCK_STRERROR(result_errno, - sebuf, sizeof(sebuf))); - break; - } - } - - /* ensure we return the intended errno to caller */ - SOCK_ERRNO_SET(result_errno); - - return n; -} - -/* - * Write data to a secure connection. - * - * Returns the number of bytes written, or a negative value (with errno - * set) upon failure. The write count could be less than requested. - * - * Note that socket-level hard failures are masked from the caller, - * instead setting conn->write_failed and storing an error message - * in conn->write_err_msg; see pqsecure_raw_write. This allows us to - * postpone reporting of write failures until we're sure no error - * message is available from the server. - * - * However, errors detected in the SSL or GSS management level are reported - * via a negative result, with message appended to conn->errorMessage. - * It's frequently unclear whether such errors should be considered read or - * write errors, so we don't attempt to postpone reporting them. - * - * The caller must still inspect errno upon failure, but only to determine - * whether to continue/retry; a message has been saved someplace in any case. - */ -ssize_t -pqsecure_write(PGconn *conn, const void *ptr, size_t len) -{ - ssize_t n; - -#ifdef USE_SSL - if (conn->ssl_in_use) - { - n = pgtls_write(conn, ptr, len); - } - else -#endif -#ifdef ENABLE_GSS - if (conn->gssenc) - { - n = pg_GSS_write(conn, ptr, len); - } - else -#endif - { - n = pqsecure_raw_write(conn, ptr, len); - } - - return n; -} - -/* - * Low-level implementation of pqsecure_write. - * - * This is used directly for an unencrypted connection. For encrypted - * connections, this does the physical I/O on behalf of pgtls_write or - * pg_GSS_write. - * - * This function reports failure (i.e., returns a negative result) only - * for retryable errors such as EINTR. Looping for such cases is to be - * handled at some outer level, maybe all the way up to the application. - * For hard failures, we set conn->write_failed and store an error message - * in conn->write_err_msg, but then claim to have written the data anyway. - * This is because we don't want to report write failures so long as there - * is a possibility of reading from the server and getting an error message - * that could explain why the connection dropped. Many TCP stacks have - * race conditions such that a write failure may or may not be reported - * before all incoming data has been read. - * - * Note that this error behavior happens below the SSL management level when - * we are using SSL. That's because at least some versions of OpenSSL are - * too quick to report a write failure when there's still a possibility to - * get a more useful error from the server. - */ -ssize_t -pqsecure_raw_write(PGconn *conn, const void *ptr, size_t len) -{ - ssize_t n; - int flags = 0; - int result_errno = 0; - char msgbuf[1024]; - char sebuf[PG_STRERROR_R_BUFLEN]; - - DECLARE_SIGPIPE_INFO(spinfo); - - /* - * If we already had a write failure, we will never again try to send data - * on that connection. Even if the kernel would let us, we've probably - * lost message boundary sync with the server. conn->write_failed - * therefore persists until the connection is reset, and we just discard - * all data presented to be written. - */ - if (conn->write_failed) - return len; - -#ifdef MSG_NOSIGNAL - if (conn->sigpipe_flag) - flags |= MSG_NOSIGNAL; - -retry_masked: -#endif /* MSG_NOSIGNAL */ - - DISABLE_SIGPIPE(conn, spinfo, return -1); - - n = send(conn->sock, ptr, len, flags); - - if (n < 0) - { - result_errno = SOCK_ERRNO; - - /* - * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available - * on this machine. So, clear sigpipe_flag so we don't try the flag - * again, and retry the send(). - */ -#ifdef MSG_NOSIGNAL - if (flags != 0 && result_errno == EINVAL) - { - conn->sigpipe_flag = false; - flags = 0; - goto retry_masked; - } -#endif /* MSG_NOSIGNAL */ - - /* Set error message if appropriate */ - switch (result_errno) - { -#ifdef EAGAIN - case EAGAIN: -#endif -#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - case EWOULDBLOCK: -#endif - case EINTR: - /* no error message, caller is expected to retry */ - break; - - case EPIPE: - /* Set flag for EPIPE */ - REMEMBER_EPIPE(spinfo, true); - - /* FALL THRU */ - - case ECONNRESET: - conn->write_failed = true; - /* Store error message in conn->write_err_msg, if possible */ - /* (strdup failure is OK, we'll cope later) */ - snprintf(msgbuf, sizeof(msgbuf), - libpq_gettext("server closed the connection unexpectedly\n" - "\tThis probably means the server terminated abnormally\n" - "\tbefore or while processing the request.")); - /* keep newline out of translated string */ - strlcat(msgbuf, "\n", sizeof(msgbuf)); - conn->write_err_msg = strdup(msgbuf); - /* Now claim the write succeeded */ - n = len; - break; - - default: - conn->write_failed = true; - /* Store error message in conn->write_err_msg, if possible */ - /* (strdup failure is OK, we'll cope later) */ - snprintf(msgbuf, sizeof(msgbuf), - libpq_gettext("could not send data to server: %s"), - SOCK_STRERROR(result_errno, - sebuf, sizeof(sebuf))); - /* keep newline out of translated string */ - strlcat(msgbuf, "\n", sizeof(msgbuf)); - conn->write_err_msg = strdup(msgbuf); - /* Now claim the write succeeded */ - n = len; - break; - } - } - - RESTORE_SIGPIPE(conn, spinfo); - - /* ensure we return the intended errno to caller */ - SOCK_ERRNO_SET(result_errno); - - return n; -} /* Dummy versions of SSL info functions, when built without SSL support */ #ifndef USE_SSL @@ -507,90 +183,3 @@ PQgssEncInUse(PGconn *conn) } #endif /* ENABLE_GSS */ - - -#if !defined(WIN32) - -/* - * Block SIGPIPE for this thread. This prevents send()/write() from exiting - * the application. - */ -int -pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending) -{ - sigset_t sigpipe_sigset; - sigset_t sigset; - - sigemptyset(&sigpipe_sigset); - sigaddset(&sigpipe_sigset, SIGPIPE); - - /* Block SIGPIPE and save previous mask for later reset */ - SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset)); - if (SOCK_ERRNO) - return -1; - - /* We can have a pending SIGPIPE only if it was blocked before */ - if (sigismember(osigset, SIGPIPE)) - { - /* Is there a pending SIGPIPE? */ - if (sigpending(&sigset) != 0) - return -1; - - if (sigismember(&sigset, SIGPIPE)) - *sigpipe_pending = true; - else - *sigpipe_pending = false; - } - else - *sigpipe_pending = false; - - return 0; -} - -/* - * Discard any pending SIGPIPE and reset the signal mask. - * - * Note: we are effectively assuming here that the C library doesn't queue - * up multiple SIGPIPE events. If it did, then we'd accidentally leave - * ours in the queue when an event was already pending and we got another. - * As long as it doesn't queue multiple events, we're OK because the caller - * can't tell the difference. - * - * The caller should say got_epipe = false if it is certain that it - * didn't get an EPIPE error; in that case we'll skip the clear operation - * and things are definitely OK, queuing or no. If it got one or might have - * gotten one, pass got_epipe = true. - * - * We do not want this to change errno, since if it did that could lose - * the error code from a preceding send(). We essentially assume that if - * we were able to do pq_block_sigpipe(), this can't fail. - */ -void -pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe) -{ - int save_errno = SOCK_ERRNO; - int signo; - sigset_t sigset; - - /* Clear SIGPIPE only if none was pending */ - if (got_epipe && !sigpipe_pending) - { - if (sigpending(&sigset) == 0 && - sigismember(&sigset, SIGPIPE)) - { - sigset_t sigpipe_sigset; - - sigemptyset(&sigpipe_sigset); - sigaddset(&sigpipe_sigset, SIGPIPE); - - sigwait(&sigpipe_sigset, &signo); - } - } - - /* Restore saved block mask */ - pthread_sigmask(SIG_SETMASK, osigset, NULL); - - SOCK_ERRNO_SET(save_errno); -} - -#endif /* !WIN32 */ diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 7888199b0d..1314663213 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -81,6 +81,7 @@ typedef struct #endif #endif /* USE_OPENSSL */ +#include "common/io_stream.h" #include "common/pg_prng.h" /* @@ -456,6 +457,7 @@ struct pg_conn PGcmdQueueEntry *cmd_queue_recycle; /* Connection data */ + IoStream *io_stream; pgsocket sock; /* FD for socket, PGINVALID_SOCKET if * unconnected */ SockAddr laddr; /* Local address */ @@ -753,11 +755,6 @@ extern int pqWriteReady(PGconn *conn); extern int pqsecure_initialize(PGconn *, bool, bool); extern PostgresPollingStatusType pqsecure_open_client(PGconn *); -extern void pqsecure_close(PGconn *); -extern ssize_t pqsecure_read(PGconn *, void *ptr, size_t len); -extern ssize_t pqsecure_write(PGconn *, const void *ptr, size_t len); -extern ssize_t pqsecure_raw_read(PGconn *, void *ptr, size_t len); -extern ssize_t pqsecure_raw_write(PGconn *, const void *ptr, size_t len); #if !defined(WIN32) extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending); @@ -793,34 +790,6 @@ extern int pgtls_init(PGconn *conn, bool do_ssl, bool do_crypto); */ extern PostgresPollingStatusType pgtls_open_client(PGconn *conn); -/* - * Close SSL connection. - */ -extern void pgtls_close(PGconn *conn); - -/* - * Read data from a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -extern ssize_t pgtls_read(PGconn *conn, void *ptr, size_t len); - -/* - * Is there unread data waiting in the SSL read buffer? - */ -extern bool pgtls_read_pending(PGconn *conn); - -/* - * Write data to a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -extern ssize_t pgtls_write(PGconn *conn, const void *ptr, size_t len); - /* * Get the hash of the server certificate, for SCRAM channel binding type * tls-server-end-point. @@ -851,13 +820,6 @@ extern int pgtls_verify_peer_name_matches_certificate_guts(PGconn *conn, * Establish a GSSAPI-encrypted connection. */ extern PostgresPollingStatusType pqsecure_open_gss(PGconn *conn); - -/* - * Read and write functions for GSSAPI-encrypted connections, with internal - * buffering to handle nonblocking sockets. - */ -extern ssize_t pg_GSS_write(PGconn *conn, const void *ptr, size_t len); -extern ssize_t pg_GSS_read(PGconn *conn, void *ptr, size_t len); #endif /* === in fe-trace.c === */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 46df01cc8d..f2013f014a 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -137,7 +137,7 @@ sub mkvcbuild our @pgcommonallfiles = qw( archive.c base64.c binaryheap.c checksum_helper.c compression.c config_info.c controldata_utils.c d2s.c encnames.c exec.c - f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c + f2s.c file_perm.c file_utils.c hashfn.c io_stream.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c pg_get_line.c pg_lzcompress.c pg_prng.c pgfnames.c psprintf.c relpath.c rmtree.c saslprep.c scram-common.c string.c stringinfo.c -- 2.42.0 [application/octet-stream] v1-0005-DO-NOT-MERGE-enable-compression-for-CI.patch (3.3K, ../../CACzsqT4cJG0kaCbz24Sd=GAEgiQDpzU8yuD6vF25zo870+3M6g@mail.gmail.com/5-v1-0005-DO-NOT-MERGE-enable-compression-for-CI.patch) download | inline diff: From 9e618ebde3e1ab78ea768c6261db6b54aae4f0b4 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Sun, 17 Dec 2023 16:09:23 -0600 Subject: [PATCH v1 5/5] DO NOT MERGE: enable compression for CI --- src/backend/libpq/compression.c | 2 +- src/backend/utils/misc/guc_tables.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/common/zpq_stream.c | 3 ++- src/interfaces/libpq/fe-connect.c | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c index bb17a23231..35c32eb926 100644 --- a/src/backend/libpq/compression.c +++ b/src/backend/libpq/compression.c @@ -17,7 +17,7 @@ #include "utils/guc_hooks.h" /* GUC variable containing the allowed compression algorithms list (separated by semicolon) */ -char *libpq_compress_algorithms = "off"; +char *libpq_compress_algorithms = "on"; pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; size_t libpq_n_compressors = 0; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6b805e9a7f..9c0c714918 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4513,7 +4513,7 @@ struct config_string ConfigureNamesString[] = GUC_REPORT }, &libpq_compress_algorithms, - "off", + "on", check_libpq_compression, assign_libpq_compression, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 00d67cc6f6..86c40ac72f 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -74,7 +74,7 @@ # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) -#libpq_compression = off # on to allow all supported compression +#libpq_compression = on # on to allow all supported compression # methods; off to disable all compression; # semicolon separated list of algorithms to allow some diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c index 98df28eb3d..0a3a3af4c6 100644 --- a/src/common/zpq_stream.c +++ b/src/common/zpq_stream.c @@ -211,7 +211,8 @@ zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len) * has not yet been enabled) for message types that would most obviously * benefit from compression */ - if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query)) + /* force enable for testing */ + if (true || (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query))) { return zpq->compress_algs[0]; } diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index e9830bc6c7..b5a87f9061 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -399,7 +399,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Replication", "D", 5, offsetof(struct pg_conn, replication)}, - {"compression", "PGCOMPRESSION", "off", NULL, + {"compression", "PGCOMPRESSION", "on", NULL, "Libpq-compression", "", 16, offsetof(struct pg_conn, compression)}, -- 2.42.0 [application/octet-stream] v1-0002-Add-protocol-layer-compression-to-libpq.patch (119.2K, ../../CACzsqT4cJG0kaCbz24Sd=GAEgiQDpzU8yuD6vF25zo870+3M6g@mail.gmail.com/6-v1-0002-Add-protocol-layer-compression-to-libpq.patch) download | inline diff: From 6c5e0830635d0bbcce868b0d0180008e77272263 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Fri, 15 Dec 2023 12:21:08 -0600 Subject: [PATCH v1 2/5] Add protocol-layer compression to libpq Adds libpq_compression and libpq_fe_compression GUCs to coordinate algorithms between frontend and backend. Adds CompressedMessage message to send compressed data and select a particular negotiated protocol respectively. Supported compression algorithms are zlib (gzip), lz4, and zstd. --- contrib/postgres_fdw/connection.c | 26 +- doc/src/sgml/config.sgml | 26 + doc/src/sgml/libpq.sgml | 50 + doc/src/sgml/protocol.sgml | 55 + meson.build | 4 +- src/backend/libpq/Makefile | 1 + src/backend/libpq/compression.c | 127 +++ src/backend/libpq/meson.build | 1 + src/backend/libpq/pqcomm.c | 86 +- src/backend/postmaster/postmaster.c | 22 +- .../libpqwalreceiver/libpqwalreceiver.c | 15 +- src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 3 + src/bin/pgbench/pgbench.c | 17 +- src/bin/psql/command.c | 18 + src/common/Makefile | 4 +- src/common/compression.c | 145 ++- src/common/meson.build | 2 + src/common/z_stream.c | 995 ++++++++++++++++ src/common/zpq_stream.c | 1004 +++++++++++++++++ src/include/common/compression.h | 8 +- src/include/common/z_stream.h | 98 ++ src/include/common/zpq_stream.h | 91 ++ src/include/libpq/compression.h | 30 + src/include/libpq/libpq-be.h | 3 + src/include/libpq/libpq.h | 1 + src/include/libpq/protocol.h | 2 +- src/include/utils/guc_hooks.h | 2 + src/interfaces/libpq/exports.txt | 3 + src/interfaces/libpq/fe-connect.c | 123 +- src/interfaces/libpq/fe-exec.c | 15 + src/interfaces/libpq/fe-misc.c | 47 +- src/interfaces/libpq/fe-protocol3.c | 92 +- src/interfaces/libpq/libpq-fe.h | 4 + src/interfaces/libpq/libpq-int.h | 18 +- src/tools/msvc/Mkvcbuild.pm | 3 +- 36 files changed, 3016 insertions(+), 137 deletions(-) create mode 100644 src/backend/libpq/compression.c create mode 100644 src/common/z_stream.c create mode 100644 src/common/zpq_stream.c create mode 100644 src/include/common/z_stream.h create mode 100644 src/include/common/zpq_stream.h create mode 100644 src/include/libpq/compression.h diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 5800c6a9fb..82c7b17b58 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -872,11 +872,14 @@ pgfdw_get_result(PGconn *conn, const char *query) pgfdw_we_get_result = WaitEventExtensionNew("PostgresFdwGetResult"); /* Sleep until there's something to do */ - wc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_SOCKET_READABLE | - WL_EXIT_ON_PM_DEATH, - PQsocket(conn), - -1L, pgfdw_we_get_result); + if (PQreadPending(conn)) + wc = WL_SOCKET_READABLE; + else + wc = WaitLatchOrSocket(MyLatch, + WL_LATCH_SET | WL_SOCKET_READABLE | + WL_EXIT_ON_PM_DEATH, + PQsocket(conn), + -1L, pgfdw_we_get_result); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); @@ -1580,11 +1583,14 @@ pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime, PGresult **result, pgfdw_we_cleanup_result = WaitEventExtensionNew("PostgresFdwCleanupResult"); /* Sleep until there's something to do */ - wc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_SOCKET_READABLE | - WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, - PQsocket(conn), - cur_timeout, pgfdw_we_cleanup_result); + if (PQreadPending(conn)) + wc = WL_SOCKET_READABLE; + else + wc = WaitLatchOrSocket(MyLatch, + WL_LATCH_SET | WL_SOCKET_READABLE | + WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + PQsocket(conn), + cur_timeout, pgfdw_we_cleanup_result); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 44cada2b40..506a1a1945 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1066,6 +1066,32 @@ include_dir 'conf.d' </listitem> </varlistentry> + <varlistentry id="guc-libpq-compression" xreflabel="libpq_compression"> + <term><varname>libpq_compression</varname> (<type>string</type>) + <indexterm> + <primary><varname>libpq_compression</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + This parameter controls the available client-server traffic compression methods. + It allows rejecting compression requests even if it is supported by the server (for example, due to security, or CPU consumption). + The default is <literal>off</literal>, which means that no compression methods are allowed. + The value <literal>on</literal> means all supported compression methods are allowed. + For more precise control, a list of the allowed compression methods can be specified. + For example, to allow only <literal>lz4</literal> and <literal>gzip</literal>, set the setting to <literal>lz4;gzip</literal>. + The server will choose the first algorithm from the list also supported by a given client. + <literal>none</literal> is also allowed when specifying a list, and if selected means that the server + will send messages uncompressed, but may still receive compressed messages if the list includes them. + This is most useful if e.g. you want to enable compression for client-to-server traffic + but not client-to-server traffic, using e.g. <literal>none;gzip;lz4</literal> as the + parameter. Also, compression level can be specified for each method, e.g. <literal>lz4:1;gzip:2</literal> + setting will set the compression level for <literal>lz4</literal> to 1 and <literal>gzip</literal> + to 2. The default compression level for each algorithm is chosen by the underlying library. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index ed88ac001a..b5b9ecc2d6 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1354,6 +1354,56 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname </listitem> </varlistentry> + <varlistentry id="libpq-connect-compression" xreflabel="compression"> + <term><literal>compression</literal></term> + <listitem> + <para> + Request compression of libpq traffic. The client sends a request with + a list of compression algorithms. Compression can be requested by a client + by including the <literal>"compression"</literal> option in its connection string. + This can either be a boolean value to enable or disable compression + (<literal>"true"</literal>/<literal>"false"</literal>, + <literal>"on"</literal>/<literal>"off"</literal>, + <literal>"yes"</literal>/<literal>"no"</literal>, + <literal>"1"</literal>/<literal>"0"</literal>), + or an explicit list of comma-separated compression algorithms + which can optionally include compression level (<literal>"gzip;lz4:2"</literal>). + The default value is <literal>"off"</literal>. + If compression is enabled but an algorithm is not explicitly specified, + the client library sends its full list of supported algorithms. + The server sends its list of supported parameters in the <varname>libpq_compression</varname> + parameter. + Both the client and server will prefer the first algorithm in their list that is supported by + the other side. + <literal>"none"</literal> is also allowed in the list, and if selected means that the client + (or server) will send messages uncompressed, but depending on the other values it still may + receive compressed messages. + This is most useful if e.g. you want to enable compression for server-to-client traffic + but not client-to-server traffic, using e.g. <literal>"none;gzip;lz4"</literal> as the + compression parameter. + </para> + <para> + After receiving a startup packet with <varname>_pq_.libpq_compression</varname> set, the + server can send CompressedData messages referencing any of the specified algorithms, for + server-to-client traffic compression. + After receiving a parameter status message with <varname>libpq_compression</varname> set, + the client can send CompressedData messages referencing any of the specified algorithms for + client-to-server traffic compression. (Note that if the client has not requested the + <varname>_pq_.libpq_compression</varname> protocol extension in the startup packet, the + server may reject all CompressedData messages even if <varname>libpq_compression</varname> + is non-empty.) + </para> + <para> + Support for compression algorithms must be enabled when the server is compiled. + Currently, three algorithms are supported: gzip (default), lz4 (if Postgres was + configured with --with-lz4 option), and zstd (if Postgres was configured with + --with-zstd option). In all cases, streaming mode is used. + Please note that using compression together with SSL may expose extra vulnerabilities: + <ulink url="https://en.wikipedia.org/wiki/CRIME">CRIME</ulink> + </para> + </listitem> + </varlistentry> + <varlistentry id="libpq-connect-client-encoding" xreflabel="client_encoding"> <term><literal>client_encoding</literal></term> <listitem> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..700e125b3c 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -92,6 +92,15 @@ such as <command>COPY</command>. </para> + <para> + The protocol supports compressing data to reduce traffic and speed-up client-server interaction. + Compression is especially useful for importing/exporting data to/from the database using the <literal>COPY</literal> command + and for replication (both physical and logical). Compression can also reduce the server's response time + for queries returning a large amount of data (for example, JSON, BLOBs, text, ...). + Currently, three algorithms are supported: DEFLATE (if PostgreSQL was built with zlib support), + LZ4 (if PostgreSQL was built with lz4 support), and ZStandard (if PostgresSQL was built with zstd support). + </para> + <sect2 id="protocol-message-concepts"> <title>Messaging Overview</title> @@ -4115,6 +4124,52 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-message-formats-CompressedData"> + <term>CompressedData (F & B)</term> + <listitem> + <para> + + <variablelist> + <varlistentry> + <term>Byte1('z')</term> + <listitem> + <para> + Identifies the message as compressed data. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>Int32</term> + <listitem> + <para> + Length of message contents in bytes, including self. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>Int8</term> + <listitem> + <para> + Selected compression algorithm, as specified in the pg_compress_algorithm enum. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term> + Byte<replaceable>n</replaceable> + </term> + <listitem> + <para> + Compressed message data. + </para> + </listitem> + </varlistentry> + </variablelist> + + </para> + </listitem> + </varlistentry> + <varlistentry id="protocol-message-formats-CopyData"> <term>CopyData (F & B)</term> <listitem> diff --git a/meson.build b/meson.build index 52c2a37c41..99e3691589 100644 --- a/meson.build +++ b/meson.build @@ -2794,14 +2794,14 @@ frontend_common_code = declare_dependency( compile_args: ['-DFRONTEND'], include_directories: [postgres_inc], sources: generated_headers, - dependencies: [os_deps, zlib, zstd], + dependencies: [os_deps, lz4, zlib, zstd], ) backend_common_code = declare_dependency( compile_args: ['-DBUILDING_DLL'], include_directories: [postgres_inc], sources: generated_headers, - dependencies: [os_deps, zlib, zstd], + dependencies: [os_deps, lz4, zlib, zstd], ) subdir('src/common') diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile index 6d385fd6a4..f4b09c09c3 100644 --- a/src/backend/libpq/Makefile +++ b/src/backend/libpq/Makefile @@ -21,6 +21,7 @@ OBJS = \ be-fsstubs.o \ be-secure-common.o \ be-secure.o \ + compression.o \ crypt.o \ hba.o \ ifaddr.o \ diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c new file mode 100644 index 0000000000..bb17a23231 --- /dev/null +++ b/src/backend/libpq/compression.c @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------- +* + * compression.c + * Functions and variables to support backend configuration of libpq + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * src/backend/libpq/compression.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "miscadmin.h" +#include "libpq/libpq-be.h" +#include "libpq/compression.h" +#include "utils/guc_hooks.h" + +/* GUC variable containing the allowed compression algorithms list (separated by semicolon) */ +char *libpq_compress_algorithms = "off"; +pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; +size_t libpq_n_compressors = 0; + +bool +check_libpq_compression(char **newval, void **extra, GucSource source) +{ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + char *serialized_compressors; + + if (zpq_parse_compression_setting(*newval, compressors, &n_compressors) == -1) + { + GUC_check_errdetail("Cannot parse the libpq_compression setting."); + return false; + } + + if (n_compressors > 0) + { + guc_free(*newval); + serialized_compressors = zpq_serialize_compressors(compressors, n_compressors); + *newval = guc_strdup(ERROR, serialized_compressors); + pfree(serialized_compressors); + } + else + { + guc_free(*newval); + *newval = guc_strdup(ERROR, ""); + } + return true; +} + +void +assign_libpq_compression(const char *newval, void *extra) +{ + if (strlen(newval) == 0) + { + libpq_n_compressors = 0; + return; + } + zpq_parse_compression_setting(newval, libpq_compressors, &libpq_n_compressors); +} + +void +configure_libpq_compression(Port *port, const char *newval) +{ + pg_compress_specification fe_compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_fe_compressors; + + Assert(!port->zpq_stream); + + if (libpq_n_compressors == 0) + { + return; + } + + /* Init compression */ + port->zpq_stream = zpq_create(libpq_compressors, libpq_n_compressors, MyProcPort->io_stream); + if (!port->zpq_stream) + { + ereport(FATAL, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("failed to initialize the compression stream")); + } + + if (strlen(newval) == 0) + { + return; + } + + if (!zpq_deserialize_compressors(newval, fe_compressors, &n_fe_compressors)) + { + ereport(FATAL, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Cannot parse the _pq_.libpq_compression setting.")); + } + + if (MyProcPort && MyProcPort->zpq_stream) + { + pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT]; + size_t n_algorithms = 0; + + if (n_fe_compressors == 0) + { + return; + } + + /* + * Intersect client and server compressors to determine the final list + * of the supported compressors. O(N^2) is negligible because of a + * small number of the compression methods. + */ + for (size_t i = 0; i < libpq_n_compressors; i++) + { + for (size_t j = 0; j < n_fe_compressors; j++) + { + if (libpq_compressors[i].algorithm == fe_compressors[j].algorithm) + { + algorithms[n_algorithms] = libpq_compressors[i].algorithm; + n_algorithms += 1; + break; + } + } + } + + zpq_enable_compression(MyProcPort->zpq_stream, algorithms, n_algorithms); + } +} diff --git a/src/backend/libpq/meson.build b/src/backend/libpq/meson.build index 74a226c2bd..83a21a7566 100644 --- a/src/backend/libpq/meson.build +++ b/src/backend/libpq/meson.build @@ -7,6 +7,7 @@ backend_sources += files( 'be-fsstubs.c', 'be-secure-common.c', 'be-secure.c', + 'compression.c', 'crypt.c', 'hba.c', 'ifaddr.c', diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 030686cc3b..60363ec1e6 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -61,6 +61,7 @@ #include <signal.h> #include <fcntl.h> #include <grp.h> +#include <pgstat.h> #include <unistd.h> #include <sys/file.h> #include <sys/socket.h> @@ -76,7 +77,9 @@ #include "common/ip.h" #include "common/io_stream.h" +#include "common/zpq_stream.h" #include "libpq/libpq.h" +#include "libpq/pqformat.h" #include "miscadmin.h" #include "port/pg_bswap.h" #include "storage/ipc.h" @@ -84,6 +87,7 @@ #include "utils/guc_hooks.h" #include "utils/memutils.h" #include "utils/wait_event.h" +#include "utils/builtins.h" /* * Cope with the various platform-specific ways to spell TCP keepalive socket @@ -1133,11 +1137,12 @@ retry: /* -------------------------------- * pq_recvbuf - load some bytes into the input buffer * - * returns 0 if OK, EOF if trouble + * nowait parameter toggles non-blocking mode. + * returns number of read bytes, EOF if trouble * -------------------------------- */ static int -pq_recvbuf(void) +pq_recvbuf(bool nowait) { if (PqRecvPointer > 0) { @@ -1153,8 +1158,8 @@ pq_recvbuf(void) PqRecvLength = PqRecvPointer = 0; } - /* Ensure that we're in blocking mode */ - socket_set_nonblocking(false); + /* Ensure that we're in the appropriate mode */ + socket_set_nonblocking(nowait); /* Can fill buffer from PqRecvLength and upwards */ for (;;) @@ -1168,9 +1173,23 @@ pq_recvbuf(void) if (r < 0) { + if (r == ZS_DECOMPRESS_ERROR) + { + char const *msg = zpq_decompress_error(MyProcPort->zpq_stream); + + if (msg == NULL) + msg = "end of stream"; + ereport(COMMERROR, + (errcode_for_socket_access(), + errmsg("failed to decompress data: %s", msg))); + return EOF; + } if (errno == EINTR) continue; /* Ok if interrupted */ + if (nowait && (errno == EAGAIN || errno == EWOULDBLOCK)) + return 0; + /* * Careful: an ereport() that tries to write to the client would * cause recursion to here, leading to stack overflow and core @@ -1194,7 +1213,7 @@ pq_recvbuf(void) } /* r contains number of bytes read, so just incr length */ PqRecvLength += r; - return 0; + return r; } } @@ -1209,7 +1228,8 @@ pq_getbyte(void) while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } return (unsigned char) PqRecvBuffer[PqRecvPointer++]; @@ -1228,7 +1248,8 @@ pq_peekbyte(void) while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } return (unsigned char) PqRecvBuffer[PqRecvPointer]; @@ -1249,49 +1270,12 @@ pq_getbyte_if_available(unsigned char *c) Assert(PqCommReadingMsg); - if (PqRecvPointer < PqRecvLength) + if (PqRecvPointer < PqRecvLength || (r = pq_recvbuf(true)) > 0) { *c = PqRecvBuffer[PqRecvPointer++]; return 1; } - /* Put the socket into non-blocking mode */ - socket_set_nonblocking(true); - - errno = 0; - - r = io_read_with_wait(MyProcPort, c, 1); - if (r < 0) - { - /* - * Ok if no data available without blocking or interrupted (though - * EINTR really shouldn't happen with a non-blocking socket). Report - * other errors. - */ - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) - r = 0; - else - { - /* - * Careful: an ereport() that tries to write to the client would - * cause recursion to here, leading to stack overflow and core - * dump! This message must go *only* to the postmaster log. - * - * If errno is zero, assume it's EOF and let the caller complain. - */ - if (errno != 0) - ereport(COMMERROR, - (errcode_for_socket_access(), - errmsg("could not receive data from client: %m"))); - r = EOF; - } - } - else if (r == 0) - { - /* EOF detected */ - r = EOF; - } - return r; } @@ -1312,7 +1296,8 @@ pq_getbytes(char *s, size_t len) { while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } amount = PqRecvLength - PqRecvPointer; @@ -1346,7 +1331,8 @@ pq_discardbytes(size_t len) { while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } amount = PqRecvLength - PqRecvPointer; @@ -1574,7 +1560,7 @@ internal_flush(void) char *bufptr = PqSendBuffer + PqSendStart; char *bufend = PqSendBuffer + PqSendPointer; - while (bufptr < bufend) + while (bufptr < bufend || io_stream_buffered_write_data(MyProcPort->io_stream) != 0) { int rc; size_t bytes_sent; @@ -1647,7 +1633,7 @@ socket_flush_if_writable(void) int res; /* Quick exit if nothing to do */ - if (PqSendPointer == PqSendStart) + if ((PqSendPointer == PqSendStart) && (io_stream_buffered_write_data(MyProcPort->io_stream) == 0)) return 0; /* No-op if reentrant call */ @@ -1670,7 +1656,7 @@ socket_flush_if_writable(void) static bool socket_is_send_pending(void) { - return (PqSendStart < PqSendPointer); + return (PqSendStart < PqSendPointer || (io_stream_buffered_write_data(MyProcPort->io_stream) != 0)); } /* -------------------------------- diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 14122de017..be440a1e52 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -99,6 +99,7 @@ #include "common/string.h" #include "lib/ilist.h" #include "libpq/auth.h" +#include "libpq/compression.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "libpq/pqsignal.h" @@ -2200,12 +2201,15 @@ retry1: valptr), errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\"."))); } + else if (strcmp(nameptr, "_pq_.libpq_compression") == 0) + { + configure_libpq_compression(port, valptr); + } else if (strncmp(nameptr, "_pq_.", 5) == 0) { /* * Any option beginning with _pq_. is reserved for use as a - * protocol-level option, but at present no such options are - * defined. + * protocol-level option. */ unrecognized_protocol_options = lappend(unrecognized_protocol_options, pstrdup(nameptr)); @@ -4380,7 +4384,21 @@ BackendInitialize(Port *port) * already did any appropriate error reporting. */ if (status != STATUS_OK) + { + /* + * Drain the socket receive buffer before exiting to allow for a + * clean (non-rst) shutdown, which ensures the client can read + * any error messages they may have in their local receive buffer + */ + char buffer[32]; + if(port->sock != PGINVALID_SOCKET) + { + pg_set_noblock(port->sock); + while(recv(port->sock, buffer, 32, 0) > 0) {} + closesocket(port->sock); + } proc_exit(0); + } /* * Now that we have the user and database name, we can set the process diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 4152d1ddbc..e0e887b06e 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -727,12 +727,15 @@ libpqrcv_PQgetResult(PGconn *streamConn) * since we'll get interrupted by signals and can handle any * interrupts here. */ - rc = WaitLatchOrSocket(MyLatch, - WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE | - WL_LATCH_SET, - PQsocket(streamConn), - 0, - WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE); + if (PQreadPending(streamConn)) + rc = WL_SOCKET_READABLE; + else + rc = WaitLatchOrSocket(MyLatch, + WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE | + WL_LATCH_SET, + PQsocket(streamConn), + 0, + WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE); /* Interrupted? */ if (rc & WL_LATCH_SET) diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index f7c9882f7c..6b805e9a7f 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -46,6 +46,7 @@ #include "common/scram-common.h" #include "jit/jit.h" #include "libpq/auth.h" +#include "libpq/compression.h" #include "libpq/libpq.h" #include "libpq/scram.h" #include "nodes/queryjumble.h" @@ -4505,6 +4506,17 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"libpq_compression", PGC_SIGHUP, CLIENT_CONN_OTHER, + gettext_noop("Sets the list of allowed libpq compression algorithms."), + NULL, + GUC_REPORT + }, + &libpq_compress_algorithms, + "off", + check_libpq_compression, assign_libpq_compression, NULL + }, + { {"application_name", PGC_USERSET, LOGGING_WHAT, gettext_noop("Sets the application name to be reported in statistics and logs."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index cf9f283cfe..00d67cc6f6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -74,6 +74,9 @@ # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) +#libpq_compression = off # on to allow all supported compression + # methods; off to disable all compression; + # semicolon separated list of algorithms to allow some # - TCP settings - # see "man tcp" for details diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 2e1650d0ad..09d43cf960 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -7433,6 +7433,9 @@ threadRun(void *arg) int nsocks; /* number of sockets to be waited for */ pg_time_usec_t min_usec; pg_time_usec_t now = 0; /* set this only if needed */ + bool buffered_rx = false; /* true if some of the clients has + * data left in SSL/ZPQ read + * buffers */ /* * identify which client sockets should be checked for input, and @@ -7468,6 +7471,9 @@ threadRun(void *arg) */ int sock = PQsocket(st->con); + /* check if conn has buffered SSL / ZPQ read data */ + buffered_rx = buffered_rx || PQreadPending(st->con); + if (sock < 0) { pg_log_error("invalid socket: %s", PQerrorMessage(st->con)); @@ -7511,7 +7517,7 @@ threadRun(void *arg) { if (nsocks > 0) { - rc = wait_on_socket_set(sockets, min_usec); + rc = buffered_rx ? 1 : wait_on_socket_set(sockets, min_usec); } else /* nothing active, simple sleep */ { @@ -7520,7 +7526,7 @@ threadRun(void *arg) } else /* no explicit delay, wait without timeout */ { - rc = wait_on_socket_set(sockets, 0); + rc = buffered_rx ? 1 : wait_on_socket_set(sockets, 0); } if (rc < 0) @@ -7560,8 +7566,11 @@ threadRun(void *arg) pg_log_error("invalid socket: %s", PQerrorMessage(st->con)); goto done; } - - if (!socket_has_input(sockets, sock, nsocks++)) + if (PQreadPending(st->con)) + { + nsocks++; + } + else if (!socket_has_input(sockets, sock, nsocks++)) continue; } else if (st->state == CSTATE_FINISHED || diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 82cc091568..4ed6fb5d21 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -172,6 +172,7 @@ static int count_lines_in_buf(PQExpBuffer buf); static void print_with_linenumbers(FILE *output, char *lines, bool is_func); static void minimal_error_message(PGresult *res); +static void printCompressionInfo(void); static void printSSLInfo(void); static void printGSSInfo(void); static bool printPsetInfo(const char *param, printQueryOpt *popt); @@ -676,6 +677,7 @@ exec_command_conninfo(PsqlScanState scan_state, bool active_branch) printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), db, PQuser(pset.db), host, PQport(pset.db)); } + printCompressionInfo(); printSSLInfo(); printGSSInfo(); } @@ -3821,6 +3823,22 @@ connection_warnings(bool in_startup) } } +/* + * printCompressionInfo + * + * Print information about used compressor/decompressor + */ +static void +printCompressionInfo(void) +{ + char *algorithms = PQcompression(pset.db); + + if (algorithms != NULL) + { + printf(_("Compression: server: %s, client: %s\n"), PQserverCompression(pset.db), algorithms); + pfree(algorithms); + } +} /* * printSSLInfo diff --git a/src/common/Makefile b/src/common/Makefile index a5cd11c6df..7917ecbe5e 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -83,7 +83,9 @@ OBJS_COMMON = \ unicode_norm.o \ username.o \ wait_error.o \ - wchar.o + wchar.o \ + z_stream.o \ + zpq_stream.o ifeq ($(with_ssl),openssl) OBJS_COMMON += \ diff --git a/src/common/compression.c b/src/common/compression.c index ee937623f0..c2b00aaf11 100644 --- a/src/common/compression.c +++ b/src/common/compression.c @@ -36,6 +36,16 @@ #include "common/compression.h" +#ifndef FRONTEND +#define ALLOC palloc +#define STRDUP pstrdup +#define FREE pfree +#else +#define ALLOC malloc +#define STRDUP strdup +#define FREE free +#endif + static int expect_integer_value(char *keyword, char *value, pg_compress_specification *result); static bool expect_boolean_value(char *keyword, char *value, @@ -113,7 +123,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* Initial setup of result object. */ result->algorithm = algorithm; result->options = 0; - result->parse_error = NULL; + result->has_error = false; /* * Assign a default level depending on the compression method. This may @@ -128,27 +138,27 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio #ifdef USE_LZ4 result->level = 0; /* fast compression mode */ #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "LZ4"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "LZ4"); #endif break; case PG_COMPRESSION_ZSTD: #ifdef USE_ZSTD result->level = ZSTD_CLEVEL_DEFAULT; #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "ZSTD"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "ZSTD"); #endif break; case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ result->level = Z_DEFAULT_COMPRESSION; #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "gzip"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "gzip"); #endif break; } @@ -201,20 +211,20 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* Reject empty keyword. */ if (kwlen == 0) { - result->parse_error = - pstrdup(_("found empty string where a compression option was expected")); + result->has_error = true; + strlcpy(result->parse_error, _("found empty string where a compression option was expected"), 255); break; } /* Extract keyword and value as separate C strings. */ - keyword = palloc(kwlen + 1); + keyword = ALLOC(kwlen + 1); memcpy(keyword, kwstart, kwlen); keyword[kwlen] = '\0'; if (!has_value) value = NULL; else { - value = palloc(vlen + 1); + value = ALLOC(vlen + 1); memcpy(value, vstart, vlen); value[vlen] = '\0'; } @@ -240,13 +250,15 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio result->options |= PG_COMPRESSION_OPTION_LONG_DISTANCE; } else - result->parse_error = - psprintf(_("unrecognized compression option: \"%s\""), keyword); + { + result->has_error = true; + snprintf(result->parse_error, 255, _("unrecognized compression option: \"%s\""), keyword); + } /* Release memory, just to be tidy. */ - pfree(keyword); + FREE(keyword); if (value != NULL) - pfree(value); + FREE(value); /* * If we got an error or have reached the end of the string, stop. @@ -256,7 +268,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio * keyword cannot have been the end of the string, but the end of the * value might have been. */ - if (result->parse_error != NULL || + if (result->has_error || (vend == NULL ? *kwend == '\0' : *vend == '\0')) break; @@ -268,7 +280,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* * Parse 'value' as an integer and return the result. * - * If parsing fails, set result->parse_error to an appropriate message + * If parsing fails, set result->has_error and write an appropriate message to result->parse_error * and return -1. */ static int @@ -279,18 +291,17 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu if (value == NULL) { - result->parse_error = - psprintf(_("compression option \"%s\" requires a value"), - keyword); + result->has_error = true; + snprintf(result->parse_error, 255, _("compression option \"%s\" requires a value"), keyword); return -1; } ivalue = strtol(value, &ivalue_endp, 10); if (ivalue_endp == value || *ivalue_endp != '\0') { - result->parse_error = - psprintf(_("value for compression option \"%s\" must be an integer"), - keyword); + result->has_error = true; + snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be an integer"), + keyword); return -1; } return ivalue; @@ -299,8 +310,8 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu /* * Parse 'value' as a boolean and return the result. * - * If parsing fails, set result->parse_error to an appropriate message - * and return -1. The caller must check result->parse_error to determine if + * If parsing fails, set result->has_error and write an appropriate message to result->parse_error + * and return -1. The caller must check result->has_error to determine if * the call was successful. * * Valid values are: yes, no, on, off, 1, 0. @@ -327,9 +338,10 @@ expect_boolean_value(char *keyword, char *value, pg_compress_specification *resu if (pg_strcasecmp(value, "0") == 0) return false; - result->parse_error = - psprintf(_("value for compression option \"%s\" must be a Boolean value"), - keyword); + + result->has_error = true; + snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be a Boolean value"), + keyword); return false; } @@ -348,7 +360,7 @@ validate_compress_specification(pg_compress_specification *spec) int default_level = 0; /* If it didn't even parse OK, it's definitely no good. */ - if (spec->parse_error != NULL) + if (spec->has_error) return spec->parse_error; /* @@ -376,16 +388,22 @@ validate_compress_specification(pg_compress_specification *spec) break; case PG_COMPRESSION_NONE: if (spec->level != 0) - return psprintf(_("compression algorithm \"%s\" does not accept a compression level"), - get_compress_algorithm_name(spec->algorithm)); + { + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a compression level"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; + } break; } if ((spec->level < min_level || spec->level > max_level) && spec->level != default_level) - return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"), - get_compress_algorithm_name(spec->algorithm), - min_level, max_level, default_level); + { + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"), + get_compress_algorithm_name(spec->algorithm), + min_level, max_level, default_level); + return spec->parse_error; + } /* * Of the compression algorithms that we currently support, only zstd @@ -394,8 +412,9 @@ validate_compress_specification(pg_compress_specification *spec) if ((spec->options & PG_COMPRESSION_OPTION_WORKERS) != 0 && (spec->algorithm != PG_COMPRESSION_ZSTD)) { - return psprintf(_("compression algorithm \"%s\" does not accept a worker count"), - get_compress_algorithm_name(spec->algorithm)); + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a worker count"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; } /* @@ -405,13 +424,45 @@ validate_compress_specification(pg_compress_specification *spec) if ((spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) != 0 && (spec->algorithm != PG_COMPRESSION_ZSTD)) { - return psprintf(_("compression algorithm \"%s\" does not support long-distance mode"), - get_compress_algorithm_name(spec->algorithm)); + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not support long-distance mode"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; } return NULL; } +bool +supported_compression_algorithm(pg_compress_algorithm algorithm) +{ + switch (algorithm) + { + case PG_COMPRESSION_NONE: + return true; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + return true; +#else + return false; +#endif + case PG_COMPRESSION_LZ4: +#ifdef USE_LZ4 + return true; +#else + return false; +#endif + case PG_COMPRESSION_ZSTD: +#ifdef USE_ZSTD + return true; +#else + return false; +#endif + /* no default, to provoke compiler warnings if values are added */ + } + Assert(false); + return false; /* placate compiler */ +} + #ifdef FRONTEND /* @@ -440,13 +491,13 @@ parse_compress_options(const char *option, char **algorithm, char **detail) { if (result == 0) { - *algorithm = pstrdup("none"); + *algorithm = STRDUP("none"); *detail = NULL; } else { - *algorithm = pstrdup("gzip"); - *detail = pstrdup(option); + *algorithm = STRDUP("gzip"); + *detail = STRDUP(option); } return; } @@ -458,19 +509,19 @@ parse_compress_options(const char *option, char **algorithm, char **detail) sep = strchr(option, ':'); if (sep == NULL) { - *algorithm = pstrdup(option); + *algorithm = STRDUP(option); *detail = NULL; } else { char *alg; - alg = palloc((sep - option) + 1); + alg = ALLOC((sep - option) + 1); memcpy(alg, option, sep - option); alg[sep - option] = '\0'; *algorithm = alg; - *detail = pstrdup(sep + 1); + *detail = STRDUP(sep + 1); } } #endif /* FRONTEND */ diff --git a/src/common/meson.build b/src/common/meson.build index 3e819108a1..35a70da8a5 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -36,6 +36,8 @@ common_sources = files( 'username.c', 'wait_error.c', 'wchar.c', + 'z_stream.c', + 'zpq_stream.c' ) if ssl.found() diff --git a/src/common/z_stream.c b/src/common/z_stream.c new file mode 100644 index 0000000000..eb34733702 --- /dev/null +++ b/src/common/z_stream.c @@ -0,0 +1,995 @@ +/*------------------------------------------------------------------------- + * + * z_stream.c + * Functions implementing streaming compression algorithms + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/z_stream.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" +#include "pg_config.h" +#include "common/z_stream.h" +#include "utils/elog.h" + +typedef struct +{ + /* + * Id of compression algorithm. + */ + pg_compress_algorithm algorithm; + + /* + * Create new compression stream. level: compression level + */ + void *(*create_compressor) (int level); + + /* + * Create new decompression stream. + */ + void *(*create_decompressor) (); + + /* + * Decompress up to "src_size" compressed bytes from *src and write up to + * "dst_size" raw (decompressed) bytes to *dst. Number of decompressed + * bytes written to *dst is stored in *dst_processed. Number of compressed + * bytes read from *src is stored in *src_processed. + * + * Return codes: ZS_OK if no errors were encountered during decompression + * attempt. This return code does not guarantee that *src_processed > 0 or + * *dst_processed > 0. + * + * ZS_STREAM_END if encountered end of compressed data stream. + * + * ZS_DECOMPRESS_ERROR if encountered an error during decompression + * attempt. + */ + int (*decompress) (void *ds, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + + /* + * Returns true if there is some data left in internal decompression + * buffers + */ + bool (*decompress_buffered_data) (void *ds); + + /* + * Compress up to "src_size" raw (non-compressed) bytes from *src and + * write up to "dst_size" compressed bytes to *dst. Number of compressed + * bytes written to *dst is stored in *dst_processed. Number of + * non-compressed bytes read from *src is stored in *src_processed. + * + * Return codes: ZS_OK if no errors were encountered during compression + * attempt. This return code does not guarantee that *src_processed > 0 or + * *dst_processed > 0. + * + * ZS_COMPRESS_ERROR if encountered an error during compression attempt. + */ + int (*compress) (void *cs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + + /* + * Returns true if there is some data left in internal compression buffers + */ + bool (*compress_buffered_data) (void *ds); + + /* + * Free compression stream created by create_compressor function. + */ + void (*free_compressor) (void *cs); + + /* + * Free decompression stream created by create_decompressor function. + */ + void (*free_decompressor) (void *ds); + + /* + * Get compressor error message. + */ + char const *(*compress_error) (void *cs); + + /* + * Get decompressor error message. + */ + char const *(*decompress_error) (void *ds); + + int (*end_compression) (void *cs, void *dst, size_t dst_size, size_t *dst_processed); +} ZAlgorithm; + +struct ZStream +{ + ZAlgorithm const *algorithm; + void *stream; +}; + +#ifndef FRONTEND +#include "utils/palloc.h" +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define FREE(size) free(size) +#endif + +#if HAVE_LIBZSTD + +#include <stdlib.h> +#include <zstd.h> + +/* + * Maximum allowed back-reference distance, expressed as power of 2. + * This setting controls max compressor/decompressor window size. + * More details https://github.com/facebook/zstd/blob/v1.4.7/lib/zstd.h#L536 + */ +#define ZSTD_WINDOWLOG_LIMIT 23 /* set max window size to 8MB */ + + +typedef struct ZS_ZSTD_CStream +{ + ZSTD_CStream *stream; + bool has_buffered_data; + char const *error; /* error message */ +} ZS_ZSTD_CStream; + +typedef struct ZS_ZSTD_DStream +{ + ZSTD_DStream *stream; + bool has_buffered_data; + char const *error; /* error message */ +} ZS_ZSTD_DStream; + +static void * +zstd_create_compressor(int level) +{ + size_t rc; + ZS_ZSTD_CStream *c_stream = (ZS_ZSTD_CStream *) ALLOC(sizeof(ZS_ZSTD_CStream)); + + c_stream->stream = ZSTD_createCStream(); + c_stream->has_buffered_data = false; + rc = ZSTD_initCStream(c_stream->stream, level); + if (ZSTD_isError(rc)) + { + ZSTD_freeCStream(c_stream->stream); + FREE(c_stream); + return NULL; + } +#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3 + ZSTD_CCtx_setParameter(c_stream->stream, ZSTD_c_windowLog, ZSTD_WINDOWLOG_LIMIT); +#endif + c_stream->error = NULL; + return c_stream; +} + +static void * +zstd_create_decompressor() +{ + size_t rc; + ZS_ZSTD_DStream *d_stream = (ZS_ZSTD_DStream *) ALLOC(sizeof(ZS_ZSTD_DStream)); + + d_stream->stream = ZSTD_createDStream(); + d_stream->has_buffered_data = false; + rc = ZSTD_initDStream(d_stream->stream); + if (ZSTD_isError(rc)) + { + ZSTD_freeDStream(d_stream->stream); + FREE(d_stream); + return NULL; + } +#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3 + ZSTD_DCtx_setParameter(d_stream->stream, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT); +#endif + d_stream->error = NULL; + return d_stream; +} + +static int +zstd_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + size_t rc; + + in.src = src; + in.pos = 0; + in.size = src_size; + + out.dst = dst; + out.pos = 0; + out.size = dst_size; + + rc = ZSTD_decompressStream(ds->stream, &out, &in); + + *src_processed = in.pos; + *dst_processed = out.pos; + if (ZSTD_isError(rc)) + { + ds->error = ZSTD_getErrorName(rc); + return ZS_DECOMPRESS_ERROR; + } + + if (rc == 0) + { + return ZS_STREAM_END; + } + + /* + * if `output.pos == output.size`, there might be some data left within + * internal buffers + */ + ds->has_buffered_data = out.pos == out.size; + + return ZS_OK; +} + +static bool +zstd_decompress_buffered_data(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + return ds->has_buffered_data; +} + +static int +zstd_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + + in.src = src; + in.pos = 0; + in.size = src_size; + + out.dst = dst; + out.pos = 0; + out.size = dst_size; + + if (in.pos < src_size) /* Has something to compress in input buffer */ + { + size_t rc = ZSTD_compressStream(cs->stream, &out, &in); + + *dst_processed = out.pos; + *src_processed = in.pos; + if (ZSTD_isError(rc)) + { + cs->error = ZSTD_getErrorName(rc); + return ZS_COMPRESS_ERROR; + } + } + + if (in.pos == src_size) /* All data is compressed: flush internal zstd + * buffer */ + { + size_t tx_not_flushed = ZSTD_flushStream(cs->stream, &out); + + *dst_processed = out.pos; + cs->has_buffered_data = tx_not_flushed > 0; + } + else + { + cs->has_buffered_data = false; + } + + return ZS_OK; +} + +static bool +zstd_compress_buffered_data(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + return cs->has_buffered_data; +} + +static int +zstd_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + size_t tx_not_flushed; + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + ZSTD_outBuffer output; + + output.dst = dst; + output.pos = 0; + output.size = dst_size; + + do + { + tx_not_flushed = ZSTD_endStream(cs->stream, &output); + } while ((tx_not_flushed > 0) && (output.pos < output.size)); + + *dst_processed = output.pos; + + cs->has_buffered_data = tx_not_flushed > 0; + return ZS_OK; +} + +static void +zstd_free_compressor(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + if (cs != NULL) + { + ZSTD_freeCStream(cs->stream); + FREE(cs); + } +} + +static void +zstd_free_decompressor(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + if (ds != NULL) + { + ZSTD_freeDStream(ds->stream); + FREE(ds); + } +} + +static char const * +zstd_compress_error(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + return cs->error; +} + +static char const * +zstd_decompress_error(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + return ds->error; +} + +static ZAlgorithm const zstd_algorithm = { + .algorithm = PG_COMPRESSION_ZSTD, + .create_compressor = zstd_create_compressor, + .create_decompressor = zstd_create_decompressor, + .decompress = zstd_decompress, + .decompress_buffered_data = zstd_decompress_buffered_data, + .compress = zstd_compress, + .compress_buffered_data = zstd_compress_buffered_data, + .free_compressor = zstd_free_compressor, + .free_decompressor = zstd_free_decompressor, + .compress_error = zstd_compress_error, + .decompress_error = zstd_decompress_error, + .end_compression = zstd_end +}; + +#endif + +#if HAVE_LIBZ + +#include <stdlib.h> +#include <zlib.h> + + +static void * +zlib_create_compressor(int level) +{ + int rc; + z_stream *c_stream = (z_stream *) ALLOC(sizeof(z_stream)); + + memset(c_stream, 0, sizeof(*c_stream)); + rc = deflateInit(c_stream, level); + if (rc != Z_OK) + { + FREE(c_stream); + return NULL; + } + return c_stream; +} + +static void * +zlib_create_decompressor() +{ + int rc; + z_stream *d_stream = (z_stream *) ALLOC(sizeof(z_stream)); + + memset(d_stream, 0, sizeof(*d_stream)); + rc = inflateInit(d_stream); + if (rc != Z_OK) + { + FREE(d_stream); + return NULL; + } + return d_stream; +} + +static int +zlib_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *ds = (z_stream *) d_stream; + int rc; + + ds->next_in = (Bytef *) src; + ds->avail_in = src_size; + ds->next_out = (Bytef *) dst; + ds->avail_out = dst_size; + + rc = inflate(ds, Z_SYNC_FLUSH); + *src_processed = src_size - ds->avail_in; + *dst_processed = dst_size - ds->avail_out; + + if (rc == Z_STREAM_END) + { + return ZS_STREAM_END; + } + if (rc != Z_OK && rc != Z_BUF_ERROR) + { + return ZS_DECOMPRESS_ERROR; + } + + return ZS_OK; +} + +static bool +zlib_decompress_buffered_data(void *d_stream) +{ + z_stream *ds = (z_stream *) d_stream; + unsigned deflate_pending = 0; + + return deflatePending(ds, &deflate_pending, Z_NULL) > 0; +} + +static int +zlib_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *cs = (z_stream *) c_stream; + int rc PG_USED_FOR_ASSERTS_ONLY; + + cs->next_out = (Bytef *) dst; + cs->avail_out = dst_size; + cs->next_in = (Bytef *) src; + cs->avail_in = src_size; + + rc = deflate(cs, Z_SYNC_FLUSH); + Assert(rc == Z_OK); + *dst_processed = dst_size - cs->avail_out; + *src_processed = src_size - cs->avail_in; + + return ZS_OK; +} + +static bool +zlib_compress_buffered_data(void *c_stream) +{ + return false; +} + +static int +zlib_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *cs = (z_stream *) c_stream; + int rc; + + cs->next_out = (Bytef *) dst; + cs->avail_out = dst_size; + cs->next_in = NULL; + cs->avail_in = 0; + + rc = deflate(cs, Z_STREAM_END); + Assert(rc == Z_OK || rc == Z_STREAM_END); + *dst_processed = dst_size - cs->avail_out; + if (rc == Z_STREAM_END) + { + return ZS_OK; + } + + return rc; +} + +static void +zlib_free_compressor(void *c_stream) +{ + z_stream *cs = (z_stream *) c_stream; + + if (cs != NULL) + { + deflateEnd(cs); + FREE(cs); + } +} + +static void +zlib_free_decompressor(void *d_stream) +{ + z_stream *ds = (z_stream *) d_stream; + + if (ds != NULL) + { + inflateEnd(ds); + FREE(ds); + } +} + +static char const * +zlib_error(void *stream) +{ + z_stream *zs = (z_stream *) stream; + + return zs->msg; +} + +/* as with elsewhere in postgres, gzip really means zlib */ +static ZAlgorithm const zlib_algorithm = { + .algorithm = PG_COMPRESSION_GZIP, + .create_compressor = zlib_create_compressor, + .create_decompressor = zlib_create_decompressor, + .decompress = zlib_decompress, + .decompress_buffered_data = zlib_decompress_buffered_data, + .compress = zlib_compress, + .compress_buffered_data = zlib_compress_buffered_data, + .free_compressor = zlib_free_compressor, + .free_decompressor = zlib_free_decompressor, + .compress_error = zlib_error, + .decompress_error = zlib_error, + .end_compression = zlib_end +}; + +#endif + +#if USE_LZ4 +#include <lz4.h> + +#define MESSAGE_MAX_BYTES 64 * 1024 +#define RING_BUFFER_BYTES (LZ4_DECODER_RING_BUFFER_SIZE(MESSAGE_MAX_BYTES)) + +typedef struct ZS_LZ4_CStream +{ + LZ4_stream_t *stream; + int level; + size_t buf_pos; + char *last_error; + char buf[RING_BUFFER_BYTES]; +} ZS_LZ4_CStream; + +typedef struct ZS_LZ4_DStream +{ + LZ4_streamDecode_t *stream; + size_t buf_pos; + size_t read_pos; + char *last_error; + char buf[RING_BUFFER_BYTES]; +} ZS_LZ4_DStream; + +static void * +lz4_create_compressor(int level) +{ + ZS_LZ4_CStream *c_stream = (ZS_LZ4_CStream *) ALLOC(sizeof(ZS_LZ4_CStream)); + + if (c_stream == NULL) + { + return NULL; + } + c_stream->stream = LZ4_createStream(); + c_stream->level = level; + c_stream->buf_pos = 0; + if (c_stream->stream == NULL) + { + FREE(c_stream); + return NULL; + } + return c_stream; +} + +static void * +lz4_create_decompressor() +{ + ZS_LZ4_DStream *d_stream = (ZS_LZ4_DStream *) ALLOC(sizeof(ZS_LZ4_DStream)); + + if (d_stream == NULL) + { + return NULL; + } + + d_stream->stream = LZ4_createStreamDecode(); + d_stream->buf_pos = 0; + d_stream->read_pos = 0; + if (d_stream->stream == NULL) + { + FREE(d_stream); + return NULL; + } + + return d_stream; +} +char last_error[256]; + +static int +lz4_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + size_t copyable; + char *decPtr; + int decBytes; + + if (ds->read_pos < ds->buf_pos) + { + decPtr = &ds->buf[ds->read_pos]; + copyable = Min(ds->buf_pos - ds->read_pos, dst_size); + memcpy(dst, decPtr, copyable); /* read msg length */ + *dst_processed = copyable; + *src_processed = 0; + ds->read_pos += copyable; + + if (ds->read_pos == ds->buf_pos && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES) + { + ds->buf_pos = 0; + ds->read_pos = 0; + } + return ZS_OK; + } + decPtr = &ds->buf[ds->buf_pos]; + + decBytes = LZ4_decompress_safe_continue(ds->stream, src, decPtr, (int) src_size, RING_BUFFER_BYTES - ds->buf_pos); + if (decBytes < 0) + { + sprintf(last_error, "LZ4 decompression failed (src_size %ld, dst_size %ld, error: %d)", src_size, RING_BUFFER_BYTES - ds->buf_pos, decBytes); + ds->last_error = last_error; +#ifndef FRONTEND + elog(ERROR, "%s", ds->last_error); +#else + return ZS_DECOMPRESS_ERROR; +#endif + } + + copyable = Min(decBytes, dst_size); + memcpy(dst, decPtr, copyable); + + *dst_processed = copyable; + *src_processed = src_size; + + ds->buf_pos += decBytes; + ds->read_pos += copyable; + /* only reset the ring buffer after the internal buffer is drained */ + if (copyable == decBytes && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES) + { + ds->buf_pos = 0; + ds->read_pos = 0; + } + + return ZS_OK; +} + +static bool +lz4_decompress_buffered_data(void *d_stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + + return ds->read_pos < ds->buf_pos; +} + +static int +lz4_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream; + int cmpBytes; + + src_size = Min(MESSAGE_MAX_BYTES, src_size); + + memcpy((char *) (cs->buf) + cs->buf_pos, src, src_size); /* write msg length */ + + if (dst_size < LZ4_compressBound(src_size)) + { + cs->last_error = "LZ4 compression failed: buffer not big enough"; +#ifndef FRONTEND + elog(ERROR, "%s", cs->last_error); +#else + return ZS_COMPRESS_ERROR; +#endif + } + + cmpBytes = LZ4_compress_fast_continue(cs->stream, (char *) (cs->buf) + cs->buf_pos, dst, (int) src_size, (int) dst_size, cs->level); + + if (cmpBytes < 0 || cmpBytes > MESSAGE_MAX_BYTES) + { + cs->last_error = "LZ4 compression failed"; +#ifndef FRONTEND + elog(ERROR, "%s", cs->last_error); +#else + return ZS_COMPRESS_ERROR; +#endif + } + + *dst_processed = cmpBytes; + *src_processed = src_size; + + cs->buf_pos += src_size; + if (cs->buf_pos >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) + { + cs->buf_pos = 0; + } + return ZS_OK; +} + +static bool +lz4_compress_buffered_data(void *d_stream) +{ + return false; +} + + +static int +lz4_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + *dst_processed = 0; + return ZS_OK; +} + +static void +lz4_free_compressor(void *c_stream) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream; + + if (cs != NULL) + { + if (cs->stream != NULL) + { + LZ4_freeStream(cs->stream); + } + FREE(cs); + } +} + +static void +lz4_free_decompressor(void *d_stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + + if (ds != NULL) + { + if (ds->stream != NULL) + { + LZ4_freeStreamDecode(ds->stream); + } + FREE(ds); + } +} + +static char const * +lz4_compress_error(void *stream) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) stream; + + /* lz4 doesn't have any explicit API to get the error names */ + return cs->last_error; +} + +static char const * +lz4_decompress_error(void *stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) stream; + + /* lz4 doesn't have any explicit API to get the error names */ + return ds->last_error; +} + +static ZAlgorithm const lz4_algorithm = { + .algorithm = PG_COMPRESSION_LZ4, + .create_compressor = lz4_create_compressor, + .create_decompressor = lz4_create_decompressor, + .decompress = lz4_decompress, + .decompress_buffered_data = lz4_decompress_buffered_data, + .compress = lz4_compress, + .compress_buffered_data = lz4_compress_buffered_data, + .free_compressor = lz4_free_compressor, + .free_decompressor = lz4_free_decompressor, + .compress_error = lz4_compress_error, + .decompress_error = lz4_decompress_error, + .end_compression = lz4_end +}; + +#endif + +static const ZAlgorithm * +zs_find_algorithm(pg_compress_algorithm algorithm) +{ +#if HAVE_LIBZ + if (algorithm == PG_COMPRESSION_GZIP) + { + return &zlib_algorithm; + } +#endif +#if USE_LZ4 + if (algorithm == PG_COMPRESSION_LZ4) + { + return &lz4_algorithm; + } +#endif +#if HAVE_LIBZSTD + if (algorithm == PG_COMPRESSION_ZSTD) + { + return &zstd_algorithm; + } +#endif + return NULL; +} + +static int +zs_init_compressor(ZStream * zs, pg_compress_specification *spec) +{ + const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm); + + if (algorithm == NULL) + { + return -1; + } + zs->algorithm = algorithm; + zs->stream = zs->algorithm->create_compressor(spec->level); + if (zs->stream == NULL) + { + return -1; + } + return 0; +} + +static int +zs_init_decompressor(ZStream * zs, pg_compress_specification *spec) +{ + const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm); + + if (algorithm == NULL) + { + return -1; + } + zs->algorithm = algorithm; + zs->stream = zs->algorithm->create_decompressor(); + if (zs->stream == NULL) + { + return -1; + } + return 0; +} + +ZStream * +zs_create_compressor(pg_compress_specification *spec) +{ + ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream)); + + if (zs_init_compressor(zs, spec)) + { + FREE(zs); + return NULL; + } + + return zs; +} + +ZStream * +zs_create_decompressor(pg_compress_specification *spec) +{ + ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream)); + + if (zs_init_decompressor(zs, spec)) + { + FREE(zs); + return NULL; + } + + return zs; +} + +int +zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *src_processed = 0; + *dst_processed = 0; + + rc = zs->algorithm->decompress(zs->stream, + src, src_size, src_processed, + dst, dst_size, dst_processed); + + if (rc == ZS_OK || rc == ZS_INCOMPLETE_SRC || rc == ZS_STREAM_END) + { + return rc; + } + + return ZS_DECOMPRESS_ERROR; +} + +int +zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *processed = 0; + *dst_processed = 0; + + rc = zs->algorithm->compress(zs->stream, + buf, size, processed, + dst, dst_size, dst_processed); + + if (rc != ZS_OK) + { + return ZS_COMPRESS_ERROR; + } + + return rc; +} + +void +zs_compressor_free(ZStream * zs) +{ + if (zs == NULL) + { + return; + } + + if (zs->stream) + { + zs->algorithm->free_compressor(zs->stream); + } + + FREE(zs); +} + +void +zs_decompressor_free(ZStream * zs) +{ + if (zs == NULL) + { + return; + } + + if (zs->stream) + { + zs->algorithm->free_decompressor(zs->stream); + } + + FREE(zs); +} + +int +zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *dst_processed = 0; + + rc = zs->algorithm->end_compression(zs->stream, dst, dst_size, dst_processed); + + if (rc != ZS_OK) + { + return ZS_COMPRESS_ERROR; + } + + return rc; +} + +char const * +zs_compress_error(ZStream * zs) +{ + return zs->algorithm->compress_error(zs->stream); +} + +char const * +zs_decompress_error(ZStream * zs) +{ + return zs->algorithm->decompress_error(zs->stream); +} + +bool +zs_compress_buffered(ZStream * zs) +{ + return zs ? zs->algorithm->compress_buffered_data(zs->stream) : false; +} + +bool +zs_decompress_buffered(ZStream * zs) +{ + return zs ? zs->algorithm->decompress_buffered_data(zs->stream) : false; +} + +pg_compress_algorithm +zs_algorithm(ZStream * zs) +{ + return zs ? zs->algorithm->algorithm : PG_COMPRESSION_NONE; +} diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c new file mode 100644 index 0000000000..222a776933 --- /dev/null +++ b/src/common/zpq_stream.c @@ -0,0 +1,1004 @@ +/*------------------------------------------------------------------------- + * + * zpq_stream.c + * IO stream layer applying ZStream compression to libpq + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/zpq_stream.c + * + *------------------------------------------------------------------------- + */ +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif +#include <unistd.h> +#include <math.h> + +#include "common/zpq_stream.h" + +#include <common/io_stream.h> +#include <libpq/protocol.h> + +#include "pg_config.h" +#include "port/pg_bswap.h" + +/* log warnings on backend */ +#ifndef FRONTEND +#define pg_log_warning(...) elog(WARNING, __VA_ARGS__) +#else +#define pg_log_warning(...) (void)0 +#endif + +#ifndef FRONTEND +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define STRDUP(str) pstrdup(str) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define STRDUP(str) strdup(str) +#define FREE(size) free(size) +#endif + +/* ZpqBuffer size, in bytes */ +#define ZPQ_BUFFER_SIZE 8192000 + +#define ZPQ_COMPRESS_THRESHOLD 60 + +/* startup messages have no type field and therefore have a null first byte */ +#define MESSAGE_TYPE_OFFSET(msg_type) (msg_type == '\0' ? 0 : 1) + +typedef struct ZpqBuffer ZpqBuffer; + + +/* ZpqBuffer used as RX/TX buffer in ZpqStream */ +struct ZpqBuffer +{ + char buf[ZPQ_BUFFER_SIZE]; + size_t size; /* current size of buf */ + size_t pos; /* current position in buf, in range [0, size] */ +}; + +/* + * Write up to "src_size" raw (decompressed) bytes. + * Returns number of written raw bytes or error code. + * Error code is either ZPQ_COMPRESS_ERROR or error code returned by the tx function. + * In the last case number of bytes written is stored in *src_processed. + */ +static int zpq_write(IoStreamLayer * layer, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written); + +/* + * Read up to "dst_size" raw (decompressed) bytes. + * Returns number of decompressed bytes or error code. + * Error code is either ZPQ_DECOMPRESS_ERROR or error code returned by the rx function. + */ +static ssize_t zpq_read(IoStreamLayer * layer, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only); + +/* + * Return true if non-flushed data left in internal rx decompression buffer. + */ +static bool zpq_buffered_rx(ZpqStream * zpq); + +/* + * Return true if non-flushed data left in internal tx compression buffer. + */ +static bool zpq_buffered_tx(ZpqStream * zpq); + +/* + * Free stream created by zs_create function. + */ +static void zpq_free(ZpqStream * zpq); + +IoStreamProcessor zpq_processor = { + .read = (io_stream_read_func) zpq_read, + .write = (io_stream_write_func) zpq_write, + .buffered_read_data = (io_stream_predicate) zpq_buffered_rx, + .buffered_write_data = (io_stream_predicate) zpq_buffered_tx, + .destroy = (io_stream_destroy_func) zpq_free +}; + +static inline void +zpq_buf_init(ZpqBuffer * zb) +{ + zb->size = 0; + zb->pos = 0; +} + +static inline size_t +zpq_buf_left(ZpqBuffer * zb) +{ + Assert(zb->buf); + return ZPQ_BUFFER_SIZE - zb->size; +} + +static inline size_t +zpq_buf_unread(ZpqBuffer * zb) +{ + return zb->size - zb->pos; +} + +static inline char * +zpq_buf_size(ZpqBuffer * zb) +{ + return (char *) (zb->buf) + zb->size; +} + +static inline char * +zpq_buf_pos(ZpqBuffer * zb) +{ + return (char *) (zb->buf) + zb->pos; +} + +static inline void +zpq_buf_size_advance(ZpqBuffer * zb, size_t value) +{ + zb->size += value; +} + +static inline void +zpq_buf_pos_advance(ZpqBuffer * zb, size_t value) +{ + zb->pos += value; +} + +static inline void +zpq_buf_reuse(ZpqBuffer * zb) +{ + size_t unread = zpq_buf_unread(zb); + + if (unread > 5) /* can read message header, don't do anything */ + return; + if (unread == 0) + { + zb->size = 0; + zb->pos = 0; + return; + } + memmove(zb->buf, zb->buf + zb->pos, unread); + zb->size = unread; + zb->pos = 0; +} + +struct ZpqStream +{ + ZStream *c_stream; /* underlying compression stream */ + ZStream *d_stream; /* underlying decompression stream */ + + bool is_compressing; /* current compression state */ + + bool is_decompressing; /* current decompression state */ + bool reading_compressed_header; /* compression header processing + * incomplete */ + size_t rx_msg_bytes_left; /* number of bytes left to process without + * changing the decompression state */ + size_t tx_msg_bytes_left; /* number of bytes left to process without + * changing the compression state */ + + ZpqBuffer rx_in; /* buffer for unprocessed data read from next + * stream layer */ + ZpqBuffer tx_in; /* buffer for unprocessed data consumed by + * zpq_write */ + ZpqBuffer tx_out; /* buffer for processed data waiting for send + * to next stream layer */ + + pg_compress_specification *compressors; /* compressors array holds the + * available compressors to use + * for compression/decompression */ + size_t n_compressors; /* size of the compressors array */ + pg_compress_algorithm compress_algs[COMPRESSION_ALGORITHM_COUNT]; /* array of compression + * algorithms supported + * by the reciever of + * the stream */ + pg_compress_algorithm compress_alg; /* active compression algorithm */ + pg_compress_algorithm decompress_alg; /* active decompression algorithm */ +}; + +/* + * Choose the algorithm to use for the message of msg_type with msg_len. + * Returns a pg_compress_algorithm with a registered compressor, or PG_COMPRESSION_NONE if no compressor is appropriate + */ +static inline pg_compress_algorithm +zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + /* + * in theory we could choose the algorithm based on the message type + * and/or more complex heuristics, but at least for now we will just use + * the first available algorithm (which defaults to none if compression + * has not yet been enabled) for message types that would most obviously + * benefit from compression + */ + if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query)) + { + return zpq->compress_algs[0]; + } + return PG_COMPRESSION_NONE; +} + +static inline bool +zpq_should_compress(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + return zpq_choose_algorithm(zpq, msg_type, msg_len) != PG_COMPRESSION_NONE; +} + +static inline pg_compress_specification * +zpq_find_compressor(ZpqStream * zpq, pg_compress_algorithm algorithm) +{ + int i; + + for (i = 0; i < zpq->n_compressors; i++) + { + if (algorithm == zpq->compressors[i].algorithm) + return &zpq->compressors[i]; + } + return NULL; +} + +static inline bool +zpq_is_compressed_msg(char msg_type) +{ + return msg_type == PqMsg_CompressedMessage; +} + +ZpqStream * +zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream) +{ + ZpqStream *zpq; + + /* zpqStream needs at least one compressor */ + if (n_compressors == 0 || compressors == NULL) + { + return NULL; + } + zpq = (ZpqStream *) ALLOC(sizeof(ZpqStream)); + /* almost all fields should default to 0/NULL/PG_COMPRESSION_NONE */ + memset(zpq, 0, sizeof(ZpqStream)); + zpq->compressors = compressors; + zpq->n_compressors = n_compressors; + zpq_buf_init(&zpq->tx_in); + zpq_buf_init(&zpq->rx_in); + zpq_buf_init(&zpq->tx_out); + + io_stream_add_layer(stream, &zpq_processor, zpq); + + return zpq; +} + +void +zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms) +{ + Assert(n_algorithms <= COMPRESSION_ALGORITHM_COUNT); + + for (int i = 0; i < COMPRESSION_ALGORITHM_COUNT; i++) + { + if (i < n_algorithms) + { + zpq->compress_algs[i] = algorithms[i]; + } + else + { + zpq->compress_algs[i] = PG_COMPRESSION_NONE; + } + } +} + +/* Compress up to src_size bytes from *src into CompressedData and write it to the tx buffer. + * Returns ZS_OK on success, ZS_COMPRESS_ERROR if encountered a compression error. */ +static inline int +zpq_write_compressed_message(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed) +{ + size_t compressed_len; + ssize_t rc; + uint32 size; + uint8 algorithm; + + /* check if have enough space */ + if (zpq_buf_left(&zpq->tx_out) <= 6) + { + /* too little space for CompressedData, abort */ + *src_processed = 0; + return ZS_OK; + } + + compressed_len = 0; + rc = zs_write(zpq->c_stream, src, src_size, src_processed, + zpq_buf_size(&zpq->tx_out) + 6, zpq_buf_left(&zpq->tx_out) - 6, &compressed_len); + + if (compressed_len > 0) + { + /* write CompressedData type */ + *zpq_buf_size(&zpq->tx_out) = PqMsg_CompressedMessage; + size = pg_hton32(compressed_len + 5); + + memcpy(zpq_buf_size(&zpq->tx_out) + 1, &size, sizeof(uint32)); /* write msg length */ + compressed_len += 6; /* append header length to compressed data + * length */ + algorithm = zpq->compress_alg; + memcpy(zpq_buf_size(&zpq->tx_out) + 5, &algorithm, sizeof(uint8)); /* write msg algorithm */ + } + + zpq_buf_size_advance(&zpq->tx_out, compressed_len); + return rc; +} + +/* Copy the data directly from *src to the tx buffer */ +static void +zpq_write_uncompressed(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed) +{ + src_size = Min(zpq_buf_left(&zpq->tx_out), src_size); + memcpy(zpq_buf_size(&zpq->tx_out), src, src_size); + + zpq_buf_size_advance(&zpq->tx_out, src_size); + *src_processed = src_size; +} + +/* Determine if should compress the next message and change the current compression state */ +static int +zpq_toggle_compression(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + pg_compress_algorithm new_compress_alg = zpq_choose_algorithm(zpq, msg_type, msg_len); + pg_compress_specification *spec; + bool should_compress = new_compress_alg != PG_COMPRESSION_NONE; + + /* + * negative new_compress_idx indicates that we should not compress this + * message + */ + if (should_compress) + { + /* + * if the new compressor does not match the current one, change out + * the underlying z_stream + */ + if (zpq->compress_alg != new_compress_alg) + { + zs_compressor_free(zpq->c_stream); + spec = zpq_find_compressor(zpq, new_compress_alg); + if (spec == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->c_stream = zs_create_compressor(spec); + if (zpq->c_stream == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->compress_alg = new_compress_alg; + } + } + + zpq->is_compressing = should_compress; + zpq->tx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type); + return 0; +} + +/* + * Internal write function. Reads the data from *src buffer, + * determines the postgres messages type and length. + * If message matches the compression criteria, it wraps the message into + * CompressedData. Otherwise, leaves the message unchanged. + * If *src data ends with incomplete message header, this function is not + * going to read this message header. + * Returns 0 on success or error code + * Number of bytes written is stored in *processed. + */ +static int +zpq_write_internal(ZpqStream * zpq, void const *src, size_t src_size, size_t *processed) +{ + size_t src_pos = 0; + ssize_t rc; + + do + { + /* + * try to read ahead the next message types and increase + * tx_msg_bytes_left, if possible + */ + while (zpq->tx_msg_bytes_left > 0 && src_size - src_pos >= zpq->tx_msg_bytes_left + 5) + { + char msg_type = *((char *) src + src_pos + zpq->tx_msg_bytes_left); + uint32 msg_len; + + memcpy(&msg_len, (char *) src + src_pos + zpq->tx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + if (zpq_should_compress(zpq, msg_type, msg_len) != zpq->is_compressing) + { + /* + * cannot proceed further, encountered compression toggle + * point + */ + break; + } + zpq->tx_msg_bytes_left += msg_len + MESSAGE_TYPE_OFFSET(msg_type); + } + + /* + * Write CompressedData if currently is compressing or have some + * buffered data left in underlying compression stream + */ + if (zs_compress_buffered(zpq->c_stream) || (zpq->is_compressing && zpq->tx_msg_bytes_left > 0)) + { + size_t buf_processed = 0; + size_t to_compress = Min(zpq->tx_msg_bytes_left, src_size - src_pos); + + rc = zpq_write_compressed_message(zpq, (char *) src + src_pos, to_compress, &buf_processed); + src_pos += buf_processed; + zpq->tx_msg_bytes_left -= buf_processed; + + if (rc != ZS_OK) + { + *processed = src_pos; + return rc; + } + } + + /* + * If not going to compress the data from *src, just write it + * uncompressed. + */ + else if (zpq->tx_msg_bytes_left > 0) + { /* determine next message type */ + size_t copy_len = Min(src_size - src_pos, zpq->tx_msg_bytes_left); + size_t copy_processed = 0; + + zpq_write_uncompressed(zpq, (char *) src + src_pos, copy_len, ©_processed); + src_pos += copy_processed; + zpq->tx_msg_bytes_left -= copy_processed; + } + + /* + * Reached the compression toggle point, fetch next message header to + * determine compression state. + */ + else + { + char msg_type; + uint32 msg_len; + + if (src_size - src_pos < 5) + { + /* + * must return here because we can't continue without full + * message header + */ + *processed = src_pos; + return 0; + } + + msg_type = *((char *) src + src_pos); + memcpy(&msg_len, (char *) src + src_pos + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + rc = zpq_toggle_compression(zpq, msg_type, msg_len); + if (rc) + { + *processed = src_pos; + return rc; + } + } + + /* + * repeat sending while there is some data in input or internal + * compression buffer + */ + } while (src_pos < src_size && zpq_buf_left(&zpq->tx_out) > 6); + + *processed = src_pos; + return 0; +} + +int +zpq_write(IoStreamLayer * self, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written) +{ + size_t src_pos = 0; + ssize_t rc; + + /* try to process as much data as possible before calling the tx_func */ + while (zpq_buf_left(&zpq->tx_out) > 6) + { + size_t copy_len = Min(zpq_buf_left(&zpq->tx_in), src_size - src_pos); + size_t processed; + + memcpy(zpq_buf_size(&zpq->tx_in), (char *) src + src_pos, copy_len); + zpq_buf_size_advance(&zpq->tx_in, copy_len); + src_pos += copy_len; + + if (zpq_buf_unread(&zpq->tx_in) == 0 && !zs_compress_buffered(zpq->c_stream)) + { + break; + } + + processed = 0; + + rc = zpq_write_internal(zpq, zpq_buf_pos(&zpq->tx_in), zpq_buf_unread(&zpq->tx_in), &processed); + zpq_buf_pos_advance(&zpq->tx_in, processed); + zpq_buf_reuse(&zpq->tx_in); + if (rc < 0) + { + *bytes_written = src_pos; + return rc; + } + if(processed == 0) + { + break; + } + } + *bytes_written = src_pos; + + /* + * call the tx_func if have any bytes to send + */ + while (zpq_buf_unread(&zpq->tx_out)) + { + size_t count; + + rc = io_stream_next_write(self, zpq_buf_pos(&zpq->tx_out), zpq_buf_unread(&zpq->tx_out), &count); + if (!rc && count > 0) + { + zpq_buf_pos_advance(&zpq->tx_out, count); + } + else + { + zpq_buf_reuse(&zpq->tx_out); + return rc; + } + } + + zpq_buf_reuse(&zpq->tx_out); + return 0; +} + +/* Decompress bytes from RX buffer and write up to dst_len of uncompressed data to *dst. + * Returns: + * ZS_OK on success, + * ZS_STREAM_END if reached end of compressed chunk + * ZS_DECOMPRESS_ERROR if encountered a decompression error */ +static inline ssize_t +zpq_read_compressed_message(ZpqStream * zpq, char *dst, size_t dst_len, size_t *dst_processed) +{ + size_t rx_processed = 0; + ssize_t rc; + size_t read_len = Min(zpq->rx_msg_bytes_left, zpq_buf_unread(&zpq->rx_in)); + + Assert(read_len == zpq->rx_msg_bytes_left); + rc = zs_read(zpq->d_stream, zpq_buf_pos(&zpq->rx_in), read_len, &rx_processed, + dst, dst_len, dst_processed); + + zpq_buf_pos_advance(&zpq->rx_in, rx_processed); + zpq->rx_msg_bytes_left -= rx_processed; + return rc; +} + +/* Copy up to dst_len bytes from rx buffer to *dst. + * Returns amount of bytes copied. */ +static inline size_t +zpq_read_uncompressed(ZpqStream * zpq, char *dst, size_t dst_len) +{ + size_t copy_len; + + Assert(zpq_buf_unread(&zpq->rx_in) > 0); + copy_len = Min(zpq->rx_msg_bytes_left, Min(zpq_buf_unread(&zpq->rx_in), dst_len)); + + memcpy(dst, zpq_buf_pos(&zpq->rx_in), copy_len); + + zpq_buf_pos_advance(&zpq->rx_in, copy_len); + zpq->rx_msg_bytes_left -= copy_len; + return copy_len; +} + +/* Determine if should decompress the next message and + * change the current decompression state */ +static inline void +zpq_toggle_decompression(ZpqStream * zpq) +{ + uint32 msg_len; + char msg_type = *zpq_buf_pos(&zpq->rx_in); + + memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + + zpq->is_decompressing = zpq_is_compressed_msg(msg_type); + zpq->rx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type); + + if (zpq->is_decompressing) + { + /* compressed message header is no longer needed, just skip it */ + zpq_buf_pos_advance(&zpq->rx_in, 5); + zpq->rx_msg_bytes_left -= 5; + zpq->reading_compressed_header = true; + } +} + +static inline ssize_t +zpq_process_switch(ZpqStream * zpq) +{ + pg_compress_algorithm algorithm; + pg_compress_specification *spec; + + if (zpq_buf_unread(&zpq->rx_in) < 1) + { + return 0; + } + + algorithm = *zpq_buf_pos(&zpq->rx_in); + + zpq_buf_pos_advance(&zpq->rx_in, 1); + zpq->reading_compressed_header = false; + zpq->rx_msg_bytes_left -= 1; + + /* + * if the new decompressor does not match the current one, change out the + * underlying z_stream + */ + if (algorithm != zpq->decompress_alg) + { + zs_decompressor_free(zpq->d_stream); + spec = zpq_find_compressor(zpq, algorithm); + if (spec == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->d_stream = zs_create_decompressor(spec); + if (zpq->d_stream == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->decompress_alg = algorithm; + } + + return 0; +} + +ssize_t +zpq_read(IoStreamLayer * self, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only) +{ + size_t dst_pos = 0; + size_t dst_processed = 0; + ssize_t rc; + + /* Read until some data fetched */ + while (dst_pos == 0) + { + zpq_buf_reuse(&zpq->rx_in); + + if (!zpq_buffered_rx(zpq) || (zpq->is_decompressing && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left)) + { + rc = io_stream_next_read(self, zpq_buf_size(&zpq->rx_in), zpq_buf_left(&zpq->rx_in), buffered_only); + if (rc > 0) /* read fetches some data */ + { + zpq_buf_size_advance(&zpq->rx_in, rc); + } + else if (rc == 0) + { + /* got no more data; return what we have */ + return dst_pos; + } + else /* read failed */ + { + return rc; + } + } + + /* + * try to read ahead the next message types and increase + * rx_msg_bytes_left, if possible (ONLY UNCOMPRESSED MESSAGES) + */ + while (!zpq->is_decompressing && zpq->rx_msg_bytes_left > 0 && (zpq_buf_unread(&zpq->rx_in) >= zpq->rx_msg_bytes_left + 5)) + { + char msg_type; + uint32 msg_len; + + msg_type = *(zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left); + if (zpq_is_compressed_msg(msg_type)) + { + /* + * cannot proceed further, encountered compression toggle + * point + */ + break; + } + + memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4); + zpq->rx_msg_bytes_left += pg_ntoh32(msg_len) + MESSAGE_TYPE_OFFSET(msg_type); + } + + + /* + * If we are in the middle of reading a message, keep reading it until + * we reach the end at which point we need to check if we should + * toggle compression + */ + if (zpq->rx_msg_bytes_left > 0 || zs_decompress_buffered(zpq->d_stream)) + { + dst_processed = 0; + if (zpq->is_decompressing || zs_decompress_buffered(zpq->d_stream)) + { + if (zpq->reading_compressed_header) + { + zpq_process_switch(zpq); + } + if (!zs_decompress_buffered(zpq->d_stream) && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left) + { + /* + * prefer to read only the fully compressed messages or + * read if some data is buffered + */ + continue; + } + rc = zpq_read_compressed_message(zpq, dst, dst_size - dst_pos, &dst_processed); + dst_pos += dst_processed; + if (rc == ZS_STREAM_END) + { + continue; + } + if (rc != ZS_OK) + { + return rc; + } + } + else + dst_pos += zpq_read_uncompressed(zpq, dst, dst_size - dst_pos); + } + else if (zpq_buf_unread(&zpq->rx_in) >= 5) + zpq_toggle_decompression(zpq); + } + return dst_pos; +} + +bool +zpq_buffered_rx(ZpqStream * zpq) +{ + return zpq ? zpq_buf_unread(&zpq->rx_in) >= 5 || (zpq_buf_unread(&zpq->rx_in) > 0 && zpq->rx_msg_bytes_left > 0) || + zs_decompress_buffered(zpq->d_stream) : 0; +} + +bool +zpq_buffered_tx(ZpqStream * zpq) +{ + return zpq ? zpq_buf_unread(&zpq->tx_in) >= 5 || (zpq_buf_unread(&zpq->tx_in) > 0 && zpq->tx_msg_bytes_left > 0) || zpq_buf_unread(&zpq->tx_out) > 0 || + zs_compress_buffered(zpq->c_stream) : 0; +} + +void +zpq_free(ZpqStream * zpq) +{ + if (zpq) + { + if (zpq->c_stream) + { + zs_compressor_free(zpq->c_stream); + } + if (zpq->d_stream) + { + zs_decompressor_free(zpq->d_stream); + } + FREE(zpq); + } +} + +char const * +zpq_compress_error(ZpqStream * zpq) +{ + return zs_compress_error(zpq->c_stream); +} + +char const * +zpq_decompress_error(ZpqStream * zpq) +{ + return zs_decompress_error(zpq->d_stream); +} + +pg_compress_algorithm +zpq_compress_algorithm(ZpqStream * zpq) +{ + return zs_algorithm(zpq->c_stream); +} + +pg_compress_algorithm +zpq_decompress_algorithm(ZpqStream * zpq) +{ + return zs_algorithm(zpq->d_stream); +} + +char * +zpq_algorithms(ZpqStream * zpq) +{ + return zpq_serialize_compressors(zpq->compressors, zpq->n_compressors); +} + +int +zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors) +{ + int i; + + *n_compressors = 0; + memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT); + + if (pg_strcasecmp(val, "true") == 0 || + pg_strcasecmp(val, "yes") == 0 || + pg_strcasecmp(val, "on") == 0 || + pg_strcasecmp(val, "1") == 0) + { + int j = 0; + + /* + * return all available compressors, processing NONE (0) as the last + * algorithm rather than the first + */ + for (i = 1; i <= COMPRESSION_ALGORITHM_COUNT; i++) + { + if (supported_compression_algorithm(i % COMPRESSION_ALGORITHM_COUNT)) + *n_compressors += 1; + } + + for (i = 1; i <= COMPRESSION_ALGORITHM_COUNT; i++) + { + if (supported_compression_algorithm(i % COMPRESSION_ALGORITHM_COUNT)) + { + parse_compress_specification(i % COMPRESSION_ALGORITHM_COUNT, NULL, &compressors[j]); + j += 1; + } + } + return 1; + } + + if (*val == 0 || + pg_strcasecmp(val, "false") == 0 || + pg_strcasecmp(val, "no") == 0 || + pg_strcasecmp(val, "off") == 0 || + pg_strcasecmp(val, "0") == 0) + { + /* Compression is disabled */ + return 0; + } + + return zpq_deserialize_compressors(val, compressors, n_compressors) ? 1 : -1; +} + +bool +zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors) +{ + int selected_alg_mask = 0; /* bitmask of already selected + * algorithms to avoid duplicates in + * compressors */ + char *c_string_dup = STRDUP(c_string); /* following parsing can + * modify the string */ + char *p = c_string_dup; + + *n_compressors = 0; + memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT); + + while (*p != '\0') + { + char *sep = strchr(p, ';'); + char *col; + char *error_detail; + pg_compress_algorithm algorithm; + pg_compress_specification *spec = &compressors[*n_compressors]; + + if (sep != NULL) + *sep = '\0'; + + col = strchr(p, ':'); + if (col != NULL) + { + *col = '\0'; + } + if (!parse_compress_algorithm(p, &algorithm)) + { + pg_log_warning("invalid compression algorithm %s", p); + goto error; + } + + if (supported_compression_algorithm(algorithm)) + { + parse_compress_specification(algorithm, col == NULL ? NULL : col + 1, spec); + error_detail = validate_compress_specification(spec); + if (error_detail) + { + pg_log_warning("invalid compression specification: %s", error_detail); + goto error; + } + if (spec->options & PG_COMPRESSION_OPTION_WORKERS || spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) + { + pg_log_warning("streaming compression does not support workers or long distance options"); + goto error; + } + + if (selected_alg_mask & (1 << algorithm)) + { + /* duplicates are not allowed */ + pg_log_warning("duplicate algorithm %s in compressors string %s", get_compress_algorithm_name(algorithm), c_string); + goto error; + } + + *n_compressors += 1; + selected_alg_mask |= 1 << algorithm; + } + else + { + /* + * Intentionally do not return an error, as we must support + * clients/servers parsing algorithms they don't suppport mixed + * with ones they do support + */ + pg_log_warning("this build does not support compression with %s", + get_compress_algorithm_name(algorithm)); + } + + if (sep) + p = sep + 1; + else + break; + } + + /* + * Always add NONE to the list at the end to allow one-directional + * compression, but only if there are other algorithms available. If there + * are no active compressors available, we should return n_compressors = 0 + * to allow other layers to skip adding the compression processing layers + */ + if (n_compressors > 0 && !(selected_alg_mask & (1 << PG_COMPRESSION_NONE))) + { + pg_compress_specification *spec = &compressors[*n_compressors]; + + spec->algorithm = PG_COMPRESSION_NONE; + *n_compressors += 1; + } + + FREE(c_string_dup); + return true; + +error: + FREE(c_string_dup); + *n_compressors = 0; + return false; +} + +/* + * Because deserialize rejects all options, this method only needs to concern + * itself with only serializing name and level + */ +char * +zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors) +{ + char *res; + char *p; + size_t i; + size_t total_len = 0; + + if (n_compressors == 0) + { + return NULL; + } + + for (i = 0; i < n_compressors; i++) + { + size_t level_len; + + /* determine the length of the compression level string */ + level_len = compressors[i].level == 0 ? 1 : (int) floor(log10(abs(compressors[i].level))) + 1; + if (compressors[i].level < 0) + { + level_len += 1; /* add the leading "-" */ + } + + /* + * single entry looks like "alg_name:compression_level," so +2 is for + * ":" and ";" symbols (or trailing null) + */ + total_len += strlen(get_compress_algorithm_name(compressors[i].algorithm)) + level_len + 2; + } + + res = p = ALLOC(total_len); + + for (i = 0; i < n_compressors; i++) + { + p += sprintf(p, "%s:%d", get_compress_algorithm_name(compressors[i].algorithm), compressors[i].level); + if (i < n_compressors - 1) + *p++ = ';'; + } + return res; +} diff --git a/src/include/common/compression.h b/src/include/common/compression.h index c94ace6e8a..9c7d68a036 100644 --- a/src/include/common/compression.h +++ b/src/include/common/compression.h @@ -17,6 +17,7 @@ /* * These values are stored in disk, for example in files generated by pg_dump. * Create the necessary backwards compatibility layers if their order changes. + * Make sure to keep COMPRESSION_ALGORITHM_COUNT in sync if adding new values. */ typedef enum pg_compress_algorithm { @@ -26,6 +27,8 @@ typedef enum pg_compress_algorithm PG_COMPRESSION_ZSTD, } pg_compress_algorithm; +#define COMPRESSION_ALGORITHM_COUNT (PG_COMPRESSION_ZSTD + 1) + #define PG_COMPRESSION_OPTION_WORKERS (1 << 0) #define PG_COMPRESSION_OPTION_LONG_DISTANCE (1 << 1) @@ -36,7 +39,9 @@ typedef struct pg_compress_specification int level; int workers; bool long_distance; - char *parse_error; /* NULL if parsing was OK, else message */ + bool has_error; + char parse_error[255]; /* Populated with error message if + * has_error is true */ } pg_compress_specification; extern void parse_compress_options(const char *option, char **algorithm, @@ -49,5 +54,6 @@ extern void parse_compress_specification(pg_compress_algorithm algorithm, pg_compress_specification *result); extern char *validate_compress_specification(pg_compress_specification *); +extern bool supported_compression_algorithm(pg_compress_algorithm algorithm); #endif diff --git a/src/include/common/z_stream.h b/src/include/common/z_stream.h new file mode 100644 index 0000000000..e456d509e1 --- /dev/null +++ b/src/include/common/z_stream.h @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * z_stream.h + * Streaming compression algorithms + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/z_stream.h + * + *------------------------------------------------------------------------- + */ +#ifndef Z_STREAM_H +#define Z_STREAM_H + +#include <stdlib.h> + +#include "compression.h" + +#define ZS_OK (0) +#define ZS_IO_ERROR (-1) +#define ZS_DECOMPRESS_ERROR (-2) +#define ZS_COMPRESS_ERROR (-3) +#define ZS_STREAM_END (-4) +#define ZS_INCOMPLETE_SRC (-5) /* cannot decompress unless full src message + * is fetched */ + +struct ZStream; +typedef struct ZStream ZStream; + +#endif + +/* + * Create compression stream for sending compressed data. + * spec: specifications of chosen decompression algorithm + */ +extern ZStream * zs_create_compressor(pg_compress_specification *spec); + +/* + * Create decompression stream for reading compressed data. + * spec: specifications of chosen decompression algorithm + */ +extern ZStream * zs_create_decompressor(pg_compress_specification *spec); + +/* + * Read up to "size" raw (decompressed) bytes. + * Returns ZS_OK on success or error code. + * Stores bytes read from src in src_processed, bytes written to dst in dst_process. + */ +extern int zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Write up to "size" raw (decompressed) bytes. + * Returns number of written raw bytes or error code. + * Returns ZS_OK on success or error code. + * Stores bytes read from buf in processed, bytes written to dst in dst_process + */ +extern int zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Returns if there is buffered raw data remaining in the stream to compress + */ +extern bool zs_compress_buffered(ZStream * zs); + +/* + * Returns if there is buffered decompressed data remaining in the stream to read + */ +extern bool zs_decompress_buffered(ZStream * zs); + +/* + * Get decompressor error message. + */ +extern char const *zs_decompress_error(ZStream * zs); + +/* + * Get compressor error message. + */ +extern char const *zs_compress_error(ZStream * zs); + +/* + * End the compression stream. + */ +extern int zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Free stream created by zs_create_compressor function. + */ +extern void zs_compressor_free(ZStream * zs); + +/* + * Free stream created by zs_create_decompressor function. + */ +extern void zs_decompressor_free(ZStream * zs); + +/* + * Get the descriptor of chosen algorithm. + */ +extern pg_compress_algorithm zs_algorithm(ZStream * zs); diff --git a/src/include/common/zpq_stream.h b/src/include/common/zpq_stream.h new file mode 100644 index 0000000000..15570f2363 --- /dev/null +++ b/src/include/common/zpq_stream.h @@ -0,0 +1,91 @@ +/*------------------------------------------------------------------------- + * + * zpq_stream.h + * IO stream layer applying ZStream compression to libpq + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/zpq_stream.h + * + *------------------------------------------------------------------------- + */ +#include "io_stream.h" +#include "z_stream.h" + +#ifndef ZPQ_STREAM_H +#define ZPQ_STREAM_H + +#define ZPQ_FATAL_ERROR (-7) +struct ZpqStream; +typedef struct ZpqStream ZpqStream; + +#endif + +/* + * Create compression stream with rx/tx function for reading/sending compressed data. + * io_stream: IO Stream to layer on top of + * rx_data: received data (compressed data already fetched from input stream) + * rx_data_size: size of data fetched from input stream + * The returned ZpqStream can only be destroyed by destoing the IoStream with io_stream_destroy. + */ +extern ZpqStream * zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream); + +/* + * Start compressing applicable outgoing data once the connection is sufficiently set up + */ +extern void zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms); + +/* + * Get decompressor error message. + */ +extern char const *zpq_decompress_error(ZpqStream * zpq); + +/* + * Get compressor error message. + */ +extern char const *zpq_compress_error(ZpqStream * zpq); + +/* + * Get the name of the current compression algorithm. + */ +extern pg_compress_algorithm zpq_compress_algorithm(ZpqStream * zpq); + +/* + * Get the name of the current decompression algorithm. + */ +extern pg_compress_algorithm zpq_decompress_algorithm(ZpqStream * zpq); + +/* + * Parse the compression setting. + * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT + * Returns: + * - 1 if the compression setting is valid + * - 0 if the compression setting is valid but disabled + * - -1 if the compression setting is invalid + * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors. + * If no supported compressors recognized or if compression is disabled, then n_compressors is set to 0. + */ +extern int + zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors); + +/* Serialize the compressors array to string so it can be transmitted to the other side during the compression startup. + * For example, for array of two compressors (zstd, level 1), (zlib, level 2) resulting string would look like "zstd:1;zlib:2". + * Returns the resulting string. + */ +extern char + *zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors); + +/* Deserialize the compressors string received during the compression setup to a compressors array. + * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT + * Returns: + * - true if the compressors string is successfully parsed + * - false otherwise + * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors. + * If no supported compressors are recognized or c_string is empty, then n_compressors is set to 0. + */ +bool + zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors); + +/* Returns the currently enabled compression algorithms using zpq_serialize_compressors */ +char *zpq_algorithms(ZpqStream * zpq); diff --git a/src/include/libpq/compression.h b/src/include/libpq/compression.h new file mode 100644 index 0000000000..927ef42751 --- /dev/null +++ b/src/include/libpq/compression.h @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------- + * + * compression.h + * Interface to libpq/compression.c + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * src/include/libpq/compression.h + * + *------------------------------------------------------------------------- + */ +#ifndef LIBPQ_COMPRESSION_H +#define LIBPQ_COMPRESSION_H + +#include "postgres.h" +#include "libpq-be.h" +#include "common/compression.h" + +extern PGDLLIMPORT char *libpq_compress_algorithms; +extern PGDLLIMPORT pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; +extern PGDLLIMPORT size_t libpq_n_compressors; + +/* + * Enbles compression processing on the given port. + * val is the value of the _pq_.libpq_compression startup packet parameter + */ +extern void configure_libpq_compression(Port *port, const char *val); + +#endif /* LIBPQ_COMPRESSION_H */ diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 87ba7f5ea0..ec3b69351c 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -53,6 +53,8 @@ typedef struct #endif #endif /* ENABLE_SSPI */ +#include <common/zpq_stream.h> + #include "common/io_stream.h" #include "datatype/timestamp.h" #include "libpq/hba.h" @@ -161,6 +163,7 @@ typedef struct Port int remote_hostname_errcode; /* see above */ char *remote_port; /* text rep of remote port */ CAC_state canAcceptConnections; /* postmaster connection status */ + ZpqStream *zpq_stream; /* streaming compression state */ /* * Information that needs to be saved from the startup packet and passed diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h index 3612280146..01cc3911fa 100644 --- a/src/include/libpq/libpq.h +++ b/src/include/libpq/libpq.h @@ -73,6 +73,7 @@ extern void StreamClose(pgsocket sock); extern void TouchSocketFiles(void); extern void RemoveSocketFiles(void); extern void pq_init(void); +extern int pq_configure(Port *port); extern int pq_getbytes(char *s, size_t len); extern void pq_startmsgread(void); extern void pq_endmsgread(void); diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h index cc46f4b586..d9eaebbf7e 100644 --- a/src/include/libpq/protocol.h +++ b/src/include/libpq/protocol.h @@ -63,7 +63,7 @@ #define PqMsg_CopyDone 'c' #define PqMsg_CopyData 'd' - +#define PqMsg_CompressedMessage 'z' /* These are the authentication request codes sent by the backend. */ diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..ee34fbb20e 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -63,6 +63,8 @@ extern bool check_effective_io_concurrency(int *newval, void **extra, GucSource source); extern bool check_huge_page_size(int *newval, void **extra, GucSource source); extern const char *show_in_hot_standby(void); +extern bool check_libpq_compression(char **newval, void **extra, GucSource source); +extern void assign_libpq_compression(const char *newval, void *extra); extern bool check_locale_messages(char **newval, void **extra, GucSource source); extern void assign_locale_messages(const char *newval, void *extra); extern bool check_locale_monetary(char **newval, void **extra, GucSource source); diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index 850734ac96..392a6815e3 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -191,3 +191,6 @@ PQclosePrepared 188 PQclosePortal 189 PQsendClosePrepared 190 PQsendClosePortal 191 +PQcompression 192 +PQreadPending 193 +PQserverCompression 194 diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index a0f12e62af..e9830bc6c7 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -25,6 +25,7 @@ #include "common/ip.h" #include "common/link-canary.h" #include "common/scram-common.h" +#include "common/zpq_stream.h" #include "common/string.h" #include "fe-auth.h" #include "libpq-fe.h" @@ -398,6 +399,10 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Replication", "D", 5, offsetof(struct pg_conn, replication)}, + {"compression", "PGCOMPRESSION", "off", NULL, + "Libpq-compression", "", 16, + offsetof(struct pg_conn, compression)}, + {"target_session_attrs", "PGTARGETSESSIONATTRS", DefaultTargetSessionAttrs, NULL, "Target-Session-Attrs", "", 15, /* sizeof("prefer-standby") = 15 */ @@ -523,6 +528,7 @@ pqDropConnection(PGconn *conn, bool flushInput) { io_stream_destroy(conn->io_stream); conn->io_stream = NULL; + conn->zpqStream = NULL; /* Close the socket itself */ if (conn->sock != PGINVALID_SOCKET) closesocket(conn->sock); @@ -1713,6 +1719,34 @@ connectOptions2(PGconn *conn) goto oom_error; } + /* + * validate compression option + */ + if (conn->compression && conn->compression[0]) + { + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + int rc = zpq_parse_compression_setting(conn->compression, compressors, &n_compressors); + + if (rc == -1) + { + conn->status = CONNECTION_BAD; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("invalid %s value: \"%s\"\n"), + "compression", conn->compression); + return false; + } + + if (rc == 1 && n_compressors == 0) + { + conn->status = CONNECTION_BAD; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("no supported algorithms found, %s value: \"%s\"\n"), + "compression", conn->compression); + return false; + } + } + /* * validate target_session_attrs option, and set target_server_type */ @@ -3347,6 +3381,20 @@ keep_going: /* We will come back to here until there is goto error_return; } + /* + * By this point we have set up any encryption layers needed, + * so we can inject compression processing if requested. + */ + if (!conn->zpqStream && conn->n_compressors > 0) + { + conn->zpqStream = zpq_create(conn->compressors, conn->n_compressors, conn->io_stream); + if (!conn->zpqStream) + { + libpq_append_conn_error(conn, "failed to set up compression stream"); + goto error_return; + } + } + /* * Send the startup packet. * @@ -3834,14 +3882,22 @@ keep_going: /* We will come back to here until there is } else if (beresp == PqMsg_NegotiateProtocolVersion) { - if (pqGetNegotiateProtocolVersion3(conn)) + if ((res = pqProcessNegotiateProtocolVersion3(conn)) < 0) { libpq_append_conn_error(conn, "received invalid protocol negotiation message"); goto error_return; } /* OK, we read the message; mark data consumed */ conn->inStart = conn->inCursor; - goto error_return; + + if (res == 0) + { + goto error_return; + } + else + { + goto keep_going; + } } /* It is an authentication request. */ @@ -4270,6 +4326,50 @@ error_return: return PGRES_POLLING_FAILED; } +/* + * Based on server GUC libpq_compression, establishes a zpq_stream to compres + * protocol traffic. Should only be called once per connection lifecycle, as the + * zpq_stream cannot be reconfigured without restarting the connection + */ +void +pqConfigureCompression(PGconn *conn, const char *compressors_str) +{ + pg_compress_specification be_compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_be_compressors; + + zpq_deserialize_compressors(compressors_str, be_compressors, &n_be_compressors); + if (conn->zpqStream) + { + pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT]; + size_t n_algorithms = 0; + + if (n_be_compressors == 0) + { + return; + } + + /* + * Intersect client and server compressors to determine the final list + * of the supported compressors. O(N^2) is negligible because of a + * small number of the compression methods. + */ + for (size_t i = 0; i < conn->n_compressors; i++) + { + for (size_t j = 0; j < n_be_compressors; j++) + { + if (conn->compressors[i].algorithm == be_compressors[j].algorithm) + { + algorithms[n_algorithms] = conn->compressors[i].algorithm; + n_algorithms += 1; + break; + } + } + } + + zpq_enable_compression(conn->zpqStream, algorithms, n_algorithms); + } +} + /* * internal_ping @@ -4480,6 +4580,7 @@ freePGconn(PGconn *conn) free(conn->fbappname); free(conn->dbName); free(conn->replication); + free(conn->compression); free(conn->pguser); if (conn->pgpass) { @@ -7163,6 +7264,24 @@ PQuser(const PGconn *conn) return conn->pguser; } +char * +PQcompression(const PGconn *conn) +{ + if (!conn || !conn->zpqStream) + return NULL; + + return zpq_algorithms(conn->zpqStream); +} + +char * +PQserverCompression(const PGconn *conn) +{ + if (!conn) + return NULL; + + return conn->server_compression; +} + char * PQpass(const PGconn *conn) { diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index b9511df2c2..b6158d29c2 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -1185,6 +1185,15 @@ pqSaveParameterStatus(PGconn *conn, const char *name, const char *value) { conn->scram_sha_256_iterations = atoi(value); } + else if (strcmp(name, "libpq_compression") == 0) + { + if (conn->server_compression) + { + free(conn->server_compression); + } + conn->server_compression = strdup(value); + pqConfigureCompression(conn, value); + } } @@ -3962,6 +3971,12 @@ pqPipelineFlush(PGconn *conn) return 0; } +int +PQreadPending(PGconn *conn) +{ + return pqReadPending(conn); +} + /* * PQfreemem - safely frees memory allocated diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index 6772f2876d..9aeeb123b7 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -51,6 +51,8 @@ #include "pg_config_paths.h" #include "port/pg_bswap.h" +#include <common/zpq_stream.h> + static int pqPutMsgBytes(const void *buf, size_t len, PGconn *conn); static int pqSendSome(PGconn *conn, int len); static int pqSocketCheck(PGconn *conn, int forRead, int forWrite, @@ -618,6 +620,14 @@ retry3: conn->inBufSize - conn->inEnd, false); if (nread < 0) { + if (nread == ZS_DECOMPRESS_ERROR) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("decompress error: %s\n"), + zpq_decompress_error(conn->zpqStream)); + return -1; + } + switch (SOCK_ERRNO) { case EINTR: @@ -713,6 +723,14 @@ retry4: conn->inBufSize - conn->inEnd, false); if (nread < 0) { + if (nread == ZS_DECOMPRESS_ERROR) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("decompress error: %s\n"), + zpq_decompress_error(conn->zpqStream)); + return -1; + } + switch (SOCK_ERRNO) { case EINTR: @@ -822,7 +840,7 @@ pqSendSome(PGconn *conn, int len) } /* while there's still data to send */ - while (len > 0) + while (len > 0 || io_stream_buffered_write_data(conn->io_stream)) { size_t sent; int rc; @@ -883,7 +901,7 @@ pqSendSome(PGconn *conn, int len) } } - if (len > 0) + if (len > 0 || rc < 0 || io_stream_buffered_write_data(conn->io_stream)) { /* * We didn't send it all, wait till we can send more. @@ -951,7 +969,7 @@ pqSendSome(PGconn *conn, int len) int pqFlush(PGconn *conn) { - if (conn->outCount > 0) + if (conn->outCount > 0 || io_stream_buffered_write_data(conn->io_stream)) { if (conn->Pfdebug) fflush(conn->Pfdebug); @@ -976,6 +994,8 @@ pqFlush(PGconn *conn) int pqWait(int forRead, int forWrite, PGconn *conn) { + if (forRead && conn->inCursor < conn->inEnd) + return 0; return pqWaitTimed(forRead, forWrite, conn, (time_t) -1); } @@ -1030,8 +1050,9 @@ pqWriteReady(PGconn *conn) * or both. Returns >0 if one or more conditions are met, 0 if it timed * out, -1 if an error occurred. * - * If SSL is in use, the SSL buffer is checked prior to checking the socket - * for read data directly. + * If protocol layers are using buffering, the buffers are checked prior + * to checking the socket for read data directly. + * */ static int pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) @@ -1068,6 +1089,22 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) return result; } +/* + * Check if there is some data pending in stream buffers. + * Returns -1 on failure, 0 if no, 1 if yes. + */ +int +pqReadPending(PGconn *conn) +{ + if (!conn) + return -1; + + if (io_stream_buffered_read_data(conn->io_stream)) + /* short-circuit the select */ + return 1; + + return 0; +} /* * Check a file descriptor for read and/or write data, possibly waiting. diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index 8c4ec079ca..9d4642cfb4 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -74,6 +74,20 @@ pqParseInput3(PGconn *conn) */ for (;;) { + /* + * Read any available buffered data + * io_stream may be null when draining the final messages from an aborted connection + */ + if (conn->io_stream && pqReadPending(conn) && (conn->inBufSize - conn->inEnd > 0)) + { + int rc = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, conn->inBufSize - conn->inEnd, true); + + if (rc > 0) + { + conn->inEnd += rc; + } + } + /* * Try to read a message. First get the type code and length. Return * if not enough data. @@ -1404,16 +1418,18 @@ reportErrorPosition(PQExpBuffer msg, const char *query, int loc, int encoding) /* * Attempt to read a NegotiateProtocolVersion message. * Entry: 'v' message type and length have already been consumed. - * Exit: returns 0 if successfully consumed message. + * Exit: returns 0 if successfully consumed message and should exit, + * returns 1 if successfully consumed message and should continue with warning, * returns EOF if not enough data. */ int -pqGetNegotiateProtocolVersion3(PGconn *conn) +pqProcessNegotiateProtocolVersion3(PGconn *conn) { int tmp; ProtocolVersion their_version; int num; PQExpBufferData buf; + int retval = 1; if (pqGetInt(&tmp, 4, conn) != 0) return EOF; @@ -1436,9 +1452,12 @@ pqGetNegotiateProtocolVersion3(PGconn *conn) } if (their_version < conn->pversion) + { libpq_append_conn_error(conn, "protocol version not supported by server: client uses %u.%u, server supports up to %u.%u", PG_PROTOCOL_MAJOR(conn->pversion), PG_PROTOCOL_MINOR(conn->pversion), PG_PROTOCOL_MAJOR(their_version), PG_PROTOCOL_MINOR(their_version)); + retval = 0; + } if (num > 0) { appendPQExpBuffer(&conn->errorMessage, @@ -1446,14 +1465,26 @@ pqGetNegotiateProtocolVersion3(PGconn *conn) "protocol extensions not supported by server: %s", num), buf.data); appendPQExpBufferChar(&conn->errorMessage, '\n'); + + /* + * We can continue to use the connection if the server doesn't support + * compression + */ + if (num > 1 || (strcmp(buf.data, "_pq_.libpq_compression") != 0)) + { + retval = 0; + } } /* neither -- server shouldn't have sent it */ if (!(their_version < conn->pversion) && !(num > 0)) + { libpq_append_conn_error(conn, "invalid %s message", "NegotiateProtocolVersion"); + retval = 0; + } termPQExpBuffer(&buf); - return 0; + return retval; } @@ -1764,7 +1795,7 @@ pqGetCopyData3(PGconn *conn, char **buffer, int async) if (msgLength == 0) { /* Don't block if async read requested */ - if (async) + if (async && !pqReadPending(conn)) return 0; /* Need to load more data */ if (pqWait(true, false, conn) || @@ -2246,6 +2277,46 @@ pqBuildStartupPacket3(PGconn *conn, int *packetlen, return startpacket; } +/* + * Build semicolon-separated list of compression algorithms requested by client. + * It can be either explicitly specified by user in connection string, or + * include all algorithms supported by client library. + * This function returns true if the compression string is successfully parsed and + * stores a comma-separated list of algorithms in *client_compressors. + * If compression is disabled, then NULL is assigned to *client_compressors. + * Also it creates an array of compressor descriptors, each element of which corresponds to + * the corresponding algorithm name in *client_compressors list. This array is stored in PGconn + * and is used during handshake when a compression acknowledgment response is received from the server. + */ +static bool +build_compressors_list(PGconn *conn, char **client_compressors, bool build_descriptors) +{ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + + if (zpq_parse_compression_setting(conn->compression, compressors, &n_compressors) < 0) + { + return false; + } + + *client_compressors = NULL; + if (build_descriptors) + { + memcpy(conn->compressors, compressors, sizeof(compressors)); + conn->n_compressors = n_compressors; + } + + if (n_compressors == 0) + { + /* no compressors available, return */ + return true; + } + + *client_compressors = zpq_serialize_compressors(compressors, n_compressors); + + return true; +} + /* * Build a startup packet given a filled-in PGconn structure. * @@ -2292,6 +2363,19 @@ build_startup_packet(const PGconn *conn, char *packet, ADD_STARTUP_OPTION("replication", conn->replication); if (conn->pgoptions && conn->pgoptions[0]) ADD_STARTUP_OPTION("options", conn->pgoptions); + if (conn->compression && conn->compression[0]) + { + char *client_compression_algorithms; + + if (build_compressors_list((PGconn *) conn, &client_compression_algorithms, packet == NULL)) + { + if (client_compression_algorithms) + { + ADD_STARTUP_OPTION("_pq_.libpq_compression", client_compression_algorithms); + free(client_compression_algorithms); + } + } + } if (conn->send_appname) { /* Use appname if present, otherwise use fallback */ diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 97762d56f5..fb7d8a9471 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -342,6 +342,8 @@ extern char *PQhostaddr(const PGconn *conn); extern char *PQport(const PGconn *conn); extern char *PQtty(const PGconn *conn); extern char *PQoptions(const PGconn *conn); +extern char *PQcompression(const PGconn *conn); +extern char *PQserverCompression(const PGconn *conn); extern ConnStatusType PQstatus(const PGconn *conn); extern PGTransactionStatusType PQtransactionStatus(const PGconn *conn); extern const char *PQparameterStatus(const PGconn *conn, @@ -501,6 +503,8 @@ extern PGPing PQpingParams(const char *const *keywords, /* Force the write buffer to be written (or at least try) */ extern int PQflush(PGconn *conn); +extern int PQreadPending(PGconn *conn); + /* * "Fast path" interface --- not really recommended for application * use diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 1314663213..4ebd12e420 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -40,6 +40,7 @@ /* include stuff common to fe and be */ #include "libpq/pqcomm.h" +#include "common/zpq_stream.h" /* include stuff found in fe only */ #include "fe-auth-sasl.h" #include "pqexpbuffer.h" @@ -406,6 +407,16 @@ struct pg_conn char *gssdelegation; /* Try to delegate GSS credentials? (0 or 1) */ char *ssl_min_protocol_version; /* minimum TLS protocol version */ char *ssl_max_protocol_version; /* maximum TLS protocol version */ + char *compression; /* stream compression (boolean value, "any" or + * list of compression algorithms separated by + * comma) */ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; /* descriptors of + * compression + * algorithms chosen by + * client */ + unsigned n_compressors; /* size of compressors array */ + char *server_compression; /* compression settings set by server */ + char *target_session_attrs; /* desired session properties */ char *require_auth; /* name of the expected auth method */ char *load_balance_hosts; /* load balance over hosts */ @@ -621,6 +632,9 @@ struct pg_conn /* Buffer for receiving various parts of messages */ PQExpBufferData workBuffer; /* expansible string */ + + /* Compression stream */ + ZpqStream *zpqStream; }; /* PGcancel stores all data necessary to cancel a connection. A copy of this @@ -680,6 +694,7 @@ extern void pqDropConnection(PGconn *conn, bool flushInput); extern int pqPacketSend(PGconn *conn, char pack_type, const void *buf, size_t buf_len); extern bool pqGetHomeDirectory(char *buf, int bufsize); +extern void pqConfigureCompression(PGconn *conn, const char *compressors); extern pgthreadlock_t pg_g_threadlock; @@ -712,7 +727,7 @@ extern void pqParseInput3(PGconn *conn); extern int pqGetErrorNotice3(PGconn *conn, bool isError); extern void pqBuildErrorMessage3(PQExpBuffer msg, const PGresult *res, PGVerbosity verbosity, PGContextVisibility show_context); -extern int pqGetNegotiateProtocolVersion3(PGconn *conn); +extern int pqProcessNegotiateProtocolVersion3(PGconn *conn); extern int pqGetCopyData3(PGconn *conn, char **buffer, int async); extern int pqGetline3(PGconn *conn, char *s, int maxlen); extern int pqGetlineAsync3(PGconn *conn, char *buffer, int bufsize); @@ -750,6 +765,7 @@ extern int pqWaitTimed(int forRead, int forWrite, PGconn *conn, time_t finish_time); extern int pqReadReady(PGconn *conn); extern int pqWriteReady(PGconn *conn); +extern int pqReadPending(PGconn *conn); /* === in fe-secure.c === */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index f2013f014a..e419dd3db8 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -141,7 +141,8 @@ sub mkvcbuild keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c pg_get_line.c pg_lzcompress.c pg_prng.c pgfnames.c psprintf.c relpath.c rmtree.c saslprep.c scram-common.c string.c stringinfo.c - unicode_category.c unicode_norm.c username.c wait_error.c wchar.c); + unicode_category.c unicode_norm.c username.c wait_error.c wchar.c + z_stream.c zpq_stream.c); if ($solution->{options}->{openssl}) { -- 2.42.0 ^ permalink raw reply [nested|flat] 80+ messages in thread
* Re: libpq compression (part 3) @ 2023-12-19 17:02 Jacob Burroughs <[email protected]> parent: Jacob Burroughs <[email protected]> 1 sibling, 1 reply; 80+ messages in thread From: Jacob Burroughs @ 2023-12-19 17:02 UTC (permalink / raw) To: [email protected] > I believe this patch series is ready for detailed review/testing, with one caveat: as can be seen here https://cirrus-ci.com/build/6732518292979712 , the build is passing on all platforms and all tests except for the primary SSL test on Windows. One correction: I apparently missed a kerberos timeout failure on freebsd with compression enabled (being color blind the checkmark and still running colors are awfully similar, and I misread what I saw). I haven't yet successfully reproduced that one, so I may or may not need some pointers to sort it out, but I think whatever it is the fix will be small enough that the patch overall is still reviewable. ^ permalink raw reply [nested|flat] 80+ messages in thread
* Re: libpq compression (part 3) @ 2023-12-20 19:39 Jacob Burroughs <[email protected]> parent: Jacob Burroughs <[email protected]> 0 siblings, 0 replies; 80+ messages in thread From: Jacob Burroughs @ 2023-12-20 19:39 UTC (permalink / raw) To: [email protected] > One correction: I apparently missed a kerberos timeout failure on > freebsd with compression enabled (being color blind the checkmark and > still running colors are awfully similar, and I misread what I saw). > I haven't yet successfully reproduced that one, so I may or may not > need some pointers to sort it out, but I think whatever it is the fix > will be small enough that the patch overall is still reviewable. I have now sorted out all of the non-Windows build issues (and removed my stray misguided attempt at fixing the Windows issue that I hadn't intended to post the first time around). The build that is *actually* passing every platform except for the one Windows SSL test mentioned in my original message can be seen here: https://cirrus-ci.com/build/5924321042890752 Attachments: [application/octet-stream] v2-0003-Add-network-traffic-stats.patch (22.7K, ../../CACzsqT67iSKp6j+sAZiyE+Jmq12LfzPreYF4-OsgkNk-UoasYA@mail.gmail.com/2-v2-0003-Add-network-traffic-stats.patch) download | inline diff: From f897a34abc5484c96c3187634fae9b5d5faca389 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Fri, 15 Dec 2023 13:08:55 -0600 Subject: [PATCH v2 3/5] Add network traffic stats Adds pg_stat_network_traffic to monitor both bytes sent over the wire and "logical" protocol bytes, allowing for measurement of compression ratio (and SSL overhead) --- doc/src/sgml/monitoring.sgml | 100 ++++++++++++++++++++ src/backend/catalog/system_views.sql | 10 ++ src/backend/libpq/pqcomm.c | 8 ++ src/backend/utils/activity/backend_status.c | 86 +++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 50 +++++++++- src/include/catalog/pg_proc.dat | 14 ++- src/include/utils/backend_status.h | 10 ++ src/test/regress/expected/rules.out | 15 ++- 8 files changed, 285 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 4f8058d8b1..52d0a4afde 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -350,6 +350,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_stat_network_traffic</structname><indexterm><primary>pg_stat_network_traffic</primary></indexterm></entry> + <entry>One row per connection (regular and replication), showing information about + network traffic of this connection. + See <link linkend="monitoring-pg-stat-network-traffic-view"> + <structname>pg_stat_network_traffic</structname></link> for details. + </entry> + </row> + <row> <entry><structname>pg_stat_ssl</structname><indexterm><primary>pg_stat_ssl</primary></indexterm></entry> <entry>One row per connection (regular and replication), showing information about @@ -2176,6 +2185,97 @@ description | Waiting for a newly initialized WAL file to reach durable storage </sect2> + <sect2 id="monitoring-pg-stat-network-traffic-view"> + <title><structname>pg_stat_network_traffic</structname></title> + + <indexterm> + <primary>pg_stat_network_traffic</primary> + </indexterm> + + <para> + The <structname>pg_stat_network_traffic</structname> view will contain one row per + backend or WAL sender process, showing statistics about network traffic on + this connection. It can be joined to <structname>pg_stat_activity</structname> + or <structname>pg_stat_replication</structname> on the + <structfield>pid</structfield> column to get more details about the + connection. + </para> + + <table id="pg-stat-network-traffic-view" xreflabel="pg_stat_network_traffic"> + <title><structname>pg_stat_network_traffic</structname> View</title> + <tgroup cols="1"> + <thead> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + Column Type + </para> + <para> + Description + </para></entry> + </row> + </thead> + + <tbody> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>pid</structfield> <type>integer</type> + </para> + <para> + Process ID of a backend or WAL sender process + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>rx_socket_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes received from the underlying socket of this connection + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>tx_socket_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes sent to the underlying socket of this connection + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>rx_pq_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes of raw libpq traffic recieved from this connection. + This can be used to estimate the overhead of SSL, which will generally + make this number smaller than <structfield>rx_socket_bytes</structfield>, + and to estimate the compression ratio when using protocol compression, + as compression will generally make this number larger than + <structfield>rx_socket_bytes</structfield>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>tx_pq_bytes</structfield> <type>integer</type> + </para> + <para> + Number of bytes of raw libpq traffic sent through this connection. + This can be used to estimate the overhead of SSL, which will generally + make this number smaller than <structfield>tx_socket_bytes</structfield>, + and to estimate the compression ratio when using protocol compression, + as compression will generally make this number larger than + <structfield>tx_socket_bytes</structfield>. + </para></entry> + </row> + </tbody> + </tgroup> + </table> + + </sect2> + <sect2 id="monitoring-pg-stat-ssl-view"> <title><structname>pg_stat_ssl</structname></title> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 11d18ed9dd..b71234b9b7 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -893,6 +893,16 @@ CREATE VIEW pg_stat_activity AS LEFT JOIN pg_database AS D ON (S.datid = D.oid) LEFT JOIN pg_authid AS U ON (S.usesysid = U.oid); +CREATE VIEW pg_stat_network_traffic AS + SELECT + S.pid, + S.rx_socket_bytes, + S.tx_socket_bytes, + S.rx_pq_bytes, + S.tx_pq_bytes + FROM pg_stat_get_activity(NULL) AS S + WHERE S.client_port IS NOT NULL; + CREATE VIEW pg_stat_replication AS SELECT S.pid, diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index ed4aad57e6..c44840881c 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -966,6 +966,8 @@ socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffer #ifdef WIN32 pgwin32_noblock = false; #endif + if (n > 0) + pgstat_report_rx_socket_traffic(n); return n; } @@ -982,6 +984,8 @@ socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size #ifdef WIN32 pgwin32_noblock = false; #endif + if (n > 0) + pgstat_report_tx_socket_traffic(n); if (n >= 0) { @@ -1009,6 +1013,8 @@ io_read_with_wait(Port *port, void *ptr, size_t len) retry: port->waitfor = WL_SOCKET_READABLE; n = io_stream_read(port->io_stream, ptr, len, false); + if (n > 0) + pgstat_report_rx_pq_traffic(n); /* In blocking mode, wait until the socket is ready */ if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) @@ -1092,6 +1098,8 @@ retry: */ rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count); *bytes_processed += count; + if (count > 0) + pgstat_report_tx_pq_traffic(count); if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) { diff --git a/src/backend/utils/activity/backend_status.c b/src/backend/utils/activity/backend_status.c index 6e734c6caf..c5712d86b8 100644 --- a/src/backend/utils/activity/backend_status.c +++ b/src/backend/utils/activity/backend_status.c @@ -338,6 +338,9 @@ pgstat_bestart(void) lbeentry.st_xact_start_timestamp = 0; lbeentry.st_databaseid = MyDatabaseId; + lbeentry.st_rx_socket_bytes = lbeentry.st_tx_socket_bytes = + lbeentry.st_rx_pq_bytes = lbeentry.st_tx_pq_bytes = 0; + /* We have userid for client-backends, wal-sender and bgworker processes */ if (lbeentry.st_backendType == B_BACKEND || lbeentry.st_backendType == B_WAL_SENDER @@ -717,6 +720,89 @@ pgstat_report_xact_timestamp(TimestampTz tstamp) PGSTAT_END_WRITE_ACTIVITY(beentry); } +/* + * Report network raw or compressed tx/rx traffic as the specified values. + */ +void +pgstat_report_rx_socket_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_rx_socket_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_tx_socket_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_tx_socket_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_rx_pq_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_rx_pq_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + +void +pgstat_report_tx_pq_traffic(uint64 bytes) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + + if (!pgstat_track_activities || !beentry) + return; + + /* + * Update my status entry, following the protocol of bumping + * st_changecount before and after. We use a volatile pointer here to + * ensure the compiler doesn't try to get cute. + */ + PGSTAT_BEGIN_WRITE_ACTIVITY(beentry); + + beentry->st_tx_pq_bytes += bytes; + + PGSTAT_END_WRITE_ACTIVITY(beentry); +} + /* ---------- * pgstat_read_current_status() - * diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 0cea320c00..936a2d81c3 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -304,7 +304,7 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) Datum pg_stat_get_activity(PG_FUNCTION_ARGS) { -#define PG_STAT_GET_ACTIVITY_COLS 31 +#define PG_STAT_GET_ACTIVITY_COLS 35 int num_backends = pgstat_fetch_stat_numbackends(); int curr_backend; int pid = PG_ARGISNULL(0) ? -1 : PG_GETARG_INT32(0); @@ -617,6 +617,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[30] = true; else values[30] = UInt64GetDatum(beentry->st_query_id); + values[31] = UInt64GetDatum(beentry->st_rx_socket_bytes); + values[32] = UInt64GetDatum(beentry->st_tx_socket_bytes); + values[33] = UInt64GetDatum(beentry->st_rx_pq_bytes); + values[34] = UInt64GetDatum(beentry->st_tx_pq_bytes); } else { @@ -646,6 +650,10 @@ pg_stat_get_activity(PG_FUNCTION_ARGS) nulls[28] = true; nulls[29] = true; nulls[30] = true; + nulls[31] = true; + nulls[32] = true; + nulls[33] = true; + nulls[34] = true; } tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); @@ -876,6 +884,46 @@ pg_stat_get_backend_start(PG_FUNCTION_ARGS) PG_RETURN_TIMESTAMPTZ(result); } +Datum +pg_stat_get_network_traffic(PG_FUNCTION_ARGS) +{ +#define PG_STAT_NETWORK_TRAFFIC_COLS 4 + TupleDesc tupdesc; + Datum values[PG_STAT_NETWORK_TRAFFIC_COLS]; + bool nulls[PG_STAT_NETWORK_TRAFFIC_COLS]; + int32 beid = PG_GETARG_INT32(0); + PgBackendStatus *beentry; + + if ((beentry = pgstat_get_beentry_by_backend_id(beid)) == NULL) + PG_RETURN_NULL(); + else if (!HAS_PGSTAT_PERMISSIONS(beentry->st_userid)) + PG_RETURN_NULL(); + + /* Initialise values and NULL flags arrays */ + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_NETWORK_TRAFFIC_COLS); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "rx_socket_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tx_socket_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "rx_pq_bytes", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "tx_pq_bytes", + INT8OID, -1, 0); + BlessTupleDesc(tupdesc); + + /* Fill values and NULLs */ + values[0] = UInt64GetDatum(beentry->st_rx_socket_bytes); + values[1] = UInt64GetDatum(beentry->st_tx_socket_bytes); + values[2] = UInt64GetDatum(beentry->st_rx_pq_bytes); + values[3] = UInt64GetDatum(beentry->st_tx_pq_bytes); + + /* Returns the record as Datum */ + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} Datum pg_stat_get_backend_client_addr(PG_FUNCTION_ARGS) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 916c8ec8d0..6cb8ff0387 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5440,9 +5440,9 @@ proname => 'pg_stat_get_activity', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', prorettype => 'record', proargtypes => 'int4', - proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8}', - proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', - proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id}', + proallargtypes => '{int4,oid,int4,oid,text,text,text,text,text,timestamptz,timestamptz,timestamptz,timestamptz,inet,text,int4,xid,xid,text,bool,text,text,int4,text,numeric,text,bool,text,bool,bool,int4,int8,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}', + proargnames => '{pid,datid,pid,usesysid,application_name,state,query,wait_event_type,wait_event,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,backend_type,ssl,sslversion,sslcipher,sslbits,ssl_client_dn,ssl_client_serial,ssl_issuer_dn,gss_auth,gss_princ,gss_enc,gss_delegation,leader_pid,query_id,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}', prosrc => 'pg_stat_get_activity' }, { oid => '8403', descr => 'describe wait events', proname => 'pg_get_wait_events', procost => '10', prorows => '250', @@ -5782,6 +5782,14 @@ proargnames => '{stats_reset,prefetch,hit,skip_init,skip_new,skip_fpw,skip_rep,wal_distance,block_distance,io_depth}', prosrc => 'pg_stat_get_recovery_prefetch' }, +{ oid => '9598', descr => 'statistics: information about network traffic', + proname => 'pg_stat_get_network_traffic', proisstrict => 'f', provolatile => 's', + proparallel => 'r', prorettype => 'record', proargtypes => 'int4', + proallargtypes => '{int4,int8,int8,int8,int8}', + proargmodes => '{i,o,o,o,o}', + proargnames => '{_beid,rx_socket_bytes,tx_socket_bytes,rx_pq_bytes,tx_pq_bytes}', + prosrc => 'pg_stat_get_network_traffic' }, + { oid => '2306', descr => 'statistics: information about SLRU caches', proname => 'pg_stat_get_slru', prorows => '100', proisstrict => 'f', proretset => 't', provolatile => 's', proparallel => 'r', diff --git a/src/include/utils/backend_status.h b/src/include/utils/backend_status.h index 75fc18c432..5df7bbd797 100644 --- a/src/include/utils/backend_status.h +++ b/src/include/utils/backend_status.h @@ -146,6 +146,12 @@ typedef struct PgBackendStatus /* application name; MUST be null-terminated */ char *st_appname; + /* client-server traffic information */ + uint64 st_rx_socket_bytes; + uint64 st_tx_socket_bytes; + uint64 st_rx_pq_bytes; + uint64 st_tx_pq_bytes; + /* * Current command string; MUST be null-terminated. Note that this string * possibly is truncated in the middle of a multi-byte character. As @@ -321,6 +327,10 @@ extern void pgstat_report_query_id(uint64 query_id, bool force); extern void pgstat_report_tempfile(size_t filesize); extern void pgstat_report_appname(const char *appname); extern void pgstat_report_xact_timestamp(TimestampTz tstamp); +extern void pgstat_report_rx_socket_traffic(uint64 bytes); +extern void pgstat_report_tx_socket_traffic(uint64 bytes); +extern void pgstat_report_rx_pq_traffic(uint64 bytes); +extern void pgstat_report_tx_pq_traffic(uint64 bytes); extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser); extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer, int buflen); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 05070393b9..4b50d7e9f7 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1760,7 +1760,7 @@ pg_stat_activity| SELECT s.datid, s.query_id, s.query, s.backend_type - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) LEFT JOIN pg_database d ON ((s.datid = d.oid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_all_indexes| SELECT c.oid AS relid, @@ -1877,7 +1877,7 @@ pg_stat_gssapi| SELECT pid, gss_princ AS principal, gss_enc AS encrypted, gss_delegation AS credentials_delegated - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) WHERE (client_port IS NOT NULL); pg_stat_io| SELECT backend_type, object, @@ -1898,6 +1898,13 @@ pg_stat_io| SELECT backend_type, fsync_time, stats_reset FROM pg_stat_get_io() b(backend_type, object, context, reads, read_time, writes, write_time, writebacks, writeback_time, extends, extend_time, op_bytes, hits, evictions, reuses, fsyncs, fsync_time, stats_reset); +pg_stat_network_traffic| SELECT pid, + rx_socket_bytes, + tx_socket_bytes, + rx_pq_bytes, + tx_pq_bytes + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) + WHERE (client_port IS NOT NULL); pg_stat_progress_analyze| SELECT s.pid, s.datid, d.datname, @@ -2079,7 +2086,7 @@ pg_stat_replication| SELECT s.pid, w.sync_priority, w.sync_state, w.reply_time - FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM ((pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) JOIN pg_stat_get_wal_senders() w(pid, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, write_lag, flush_lag, replay_lag, sync_priority, sync_state, reply_time) ON ((s.pid = w.pid))) LEFT JOIN pg_authid u ON ((s.usesysid = u.oid))); pg_stat_replication_slots| SELECT s.slot_name, @@ -2113,7 +2120,7 @@ pg_stat_ssl| SELECT pid, ssl_client_dn AS client_dn, ssl_client_serial AS client_serial, ssl_issuer_dn AS issuer_dn - FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id) + FROM pg_stat_get_activity(NULL::integer) s(datid, pid, usesysid, application_name, state, query, wait_event_type, wait_event, xact_start, query_start, backend_start, state_change, client_addr, client_hostname, client_port, backend_xid, backend_xmin, backend_type, ssl, sslversion, sslcipher, sslbits, ssl_client_dn, ssl_client_serial, ssl_issuer_dn, gss_auth, gss_princ, gss_enc, gss_delegation, leader_pid, query_id, rx_socket_bytes, tx_socket_bytes, rx_pq_bytes, tx_pq_bytes) WHERE (client_port IS NOT NULL); pg_stat_subscription| SELECT su.oid AS subid, su.subname, -- 2.42.0 [application/octet-stream] v2-0005-DO-NOT-MERGE-enable-compression-for-CI.patch (3.3K, ../../CACzsqT67iSKp6j+sAZiyE+Jmq12LfzPreYF4-OsgkNk-UoasYA@mail.gmail.com/3-v2-0005-DO-NOT-MERGE-enable-compression-for-CI.patch) download | inline diff: From a130f1f2ec03c8feee2d3b1885b81d27735500dd Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Sun, 17 Dec 2023 16:09:23 -0600 Subject: [PATCH v2 5/5] DO NOT MERGE: enable compression for CI --- src/backend/libpq/compression.c | 2 +- src/backend/utils/misc/guc_tables.c | 2 +- src/backend/utils/misc/postgresql.conf.sample | 2 +- src/common/zpq_stream.c | 3 ++- src/interfaces/libpq/fe-connect.c | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c index bb17a23231..35c32eb926 100644 --- a/src/backend/libpq/compression.c +++ b/src/backend/libpq/compression.c @@ -17,7 +17,7 @@ #include "utils/guc_hooks.h" /* GUC variable containing the allowed compression algorithms list (separated by semicolon) */ -char *libpq_compress_algorithms = "off"; +char *libpq_compress_algorithms = "on"; pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; size_t libpq_n_compressors = 0; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 0d36ca52f2..bd084e184a 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -4539,7 +4539,7 @@ struct config_string ConfigureNamesString[] = GUC_REPORT }, &libpq_compress_algorithms, - "off", + "on", check_libpq_compression, assign_libpq_compression, NULL }, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 0411571c02..24a4d6e05a 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -74,7 +74,7 @@ # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) -#libpq_compression = off # on to allow all supported compression +#libpq_compression = on # on to allow all supported compression # methods; off to disable all compression; # semicolon separated list of algorithms to allow some diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c index b2a47c7537..acbe070734 100644 --- a/src/common/zpq_stream.c +++ b/src/common/zpq_stream.c @@ -217,7 +217,8 @@ zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len) * has not yet been enabled) for message types that would most obviously * benefit from compression */ - if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query)) + /* force enable for testing */ + if (true || (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query))) { return zpq->compress_algs[0]; } diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 2356b9719c..92edcca0c8 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -399,7 +399,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Replication", "D", 5, offsetof(struct pg_conn, replication)}, - {"compression", "PGCOMPRESSION", "off", NULL, + {"compression", "PGCOMPRESSION", "on", NULL, "Libpq-compression", "", 16, offsetof(struct pg_conn, compression)}, -- 2.42.0 [application/octet-stream] v2-0004-Add-basic-test-for-compression-functionality.patch (6.9K, ../../CACzsqT67iSKp6j+sAZiyE+Jmq12LfzPreYF4-OsgkNk-UoasYA@mail.gmail.com/4-v2-0004-Add-basic-test-for-compression-functionality.patch) download | inline diff: From 478710fa1fb872488eb003269819da926e99a338 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Mon, 18 Dec 2023 14:15:18 -0600 Subject: [PATCH v2 4/5] Add basic test for compression functionality --- src/Makefile.global.in | 2 + src/interfaces/libpq/Makefile | 7 +- src/interfaces/libpq/meson.build | 8 +- src/interfaces/libpq/t/005_compression.pl | 95 +++++++++++++++++++++++ 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 src/interfaces/libpq/t/005_compression.pl diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 104e5de0fe..ab21065b72 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -195,9 +195,11 @@ with_ldap = @with_ldap@ with_libxml = @with_libxml@ with_libxslt = @with_libxslt@ with_llvm = @with_llvm@ +with_lz4 = @with_lz4@ with_system_tzdata = @with_system_tzdata@ with_uuid = @with_uuid@ with_zlib = @with_zlib@ +with_zstd = @with_zstd@ enable_rpath = @enable_rpath@ enable_nls = @enable_nls@ enable_debug = @enable_debug@ diff --git a/src/interfaces/libpq/Makefile b/src/interfaces/libpq/Makefile index df8a1f31b2..12dc5868b6 100644 --- a/src/interfaces/libpq/Makefile +++ b/src/interfaces/libpq/Makefile @@ -14,6 +14,9 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global export with_ssl +export with_zlib +export with_lz4 +export with_zstd PGFILEDESC = "PostgreSQL Access Library" @@ -78,9 +81,9 @@ endif # that are built correctly for use in a shlib. SHLIB_LINK_INTERNAL = -lpgcommon_shlib -lpgport_shlib ifneq ($(PORTNAME), win32) -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi_krb5 -lgss -lgssapi -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd, $(LIBS)) $(LDAP_LIBS_FE) $(PTHREAD_LIBS) else -SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) +SHLIB_LINK += $(filter -lcrypt -ldes -lcom_err -lcrypto -lk5crypto -lkrb5 -lgssapi32 -lssl -lsocket -lnsl -lresolv -lintl -lm -lz -llz4 -lzstd $(PTHREAD_LIBS), $(LIBS)) $(LDAP_LIBS_FE) endif ifeq ($(PORTNAME), win32) SHLIB_LINK += -lshell32 -lws2_32 -lsecur32 $(filter -leay32 -lssleay32 -lcomerr32 -lkrb5_32, $(LIBS)) diff --git a/src/interfaces/libpq/meson.build b/src/interfaces/libpq/meson.build index 80e6a15adf..e9b3c22a52 100644 --- a/src/interfaces/libpq/meson.build +++ b/src/interfaces/libpq/meson.build @@ -118,8 +118,14 @@ tests += { 't/002_api.pl', 't/003_load_balance_host_list.pl', 't/004_load_balance_dns.pl', + 't/005_compression.pl', ], - 'env': {'with_ssl': ssl_library}, + 'env': { + 'with_ssl': ssl_library, + 'with_zlib': zlib.found() ? 'yes' : 'no', + 'with_lz4': lz4.found() ? 'yes' : 'no', + 'with_zstd': zstd.found() ? 'yes' : 'no', + }, }, } diff --git a/src/interfaces/libpq/t/005_compression.pl b/src/interfaces/libpq/t/005_compression.pl new file mode 100644 index 0000000000..f029123d2b --- /dev/null +++ b/src/interfaces/libpq/t/005_compression.pl @@ -0,0 +1,95 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group +use strict; +use warnings; +use Config; +use PostgreSQL::Test::Utils; +use PostgreSQL::Test::Cluster; +use Test::More; + +my $withZlib = $ENV{with_zlib} eq 'yes'; +my $withLz4 = $ENV{with_lz4} eq 'yes'; +my $withZstd = $ENV{with_zstd} eq 'yes'; +if (!$withZlib && !$withLz4 && !$withZstd) +{ + plan skip_all => 'no compression methods available'; +} + +my $node = PostgreSQL::Test::Cluster->new('primary'); +$node->init; +$node->append_conf('postgresql.conf', "libpq_compression = off"); +$node->start; + +# Use a string that any reasonable compression algorithm will compress +my $compressableString = "a" x 1000; + +my $result; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=on"); +is( (split "\n", $result)[-1], "f|f", 'successfully does not compress if server-side compression is disabled'); + +$node->append_conf('postgresql.conf', "libpq_compression = on"); +$node->reload; + +my $result; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=on"); +is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with default algorithms'); + +my $test_algorithm; +if ($withZlib) +{ + $test_algorithm = "gzip"; +} +elsif($withLz4) +{ + $test_algorithm = "lz4"; +} +elsif($withZstd) +{ + $test_algorithm = "zstd"; +} +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=none;$test_algorithm"); +is( (split "\n", $result)[-1], "f|t", 'successfully compresses unidirectionally with none as first client algorithm'); + +$node->append_conf('postgresql.conf', "libpq_compression = 'none;$test_algorithm'"); +$node->reload; +$result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + "compression=on"); +is( (split "\n", $result)[-1], "t|f", 'successfully compresses unidirectionally with none as first server algorithm'); + +$node->append_conf('postgresql.conf', 'libpq_compression = on'); +$node->reload; + +SKIP: { + skip "gzip not available", 1 unless $ENV{with_zlib}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=gzip"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with gzip'); +} + +SKIP: { + skip "lz4 not available", 1 unless $ENV{with_lz4}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=lz4"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with lz4'); +} + +SKIP: { + skip "zstd not available", 1 unless $ENV{with_zstd}; + + $result = $node->safe_psql("postgres", + "SELECT '$compressableString'; SELECT rx_socket_bytes < rx_pq_bytes, tx_socket_bytes < tx_pq_bytes FROM pg_stat_network_traffic;", + connstr => $node->connstr . " compression=zstd"); + is( (split "\n", $result)[-1], "t|t", 'successfully compresses bidirectionally with zstd'); +} + +done_testing(); -- 2.42.0 [application/octet-stream] v2-0002-Add-protocol-layer-compression-to-libpq.patch (124.1K, ../../CACzsqT67iSKp6j+sAZiyE+Jmq12LfzPreYF4-OsgkNk-UoasYA@mail.gmail.com/5-v2-0002-Add-protocol-layer-compression-to-libpq.patch) download | inline diff: From 33c57d28a1733daf5b3488cf0fdb437011fdc434 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Fri, 15 Dec 2023 12:21:08 -0600 Subject: [PATCH v2 2/5] Add protocol-layer compression to libpq Adds libpq_compression and libpq_fe_compression GUCs to coordinate algorithms between frontend and backend. Adds CompressedMessage message to send compressed data and select a particular negotiated protocol respectively. Supported compression algorithms are zlib (gzip), lz4, and zstd. --- contrib/postgres_fdw/connection.c | 26 +- doc/src/sgml/config.sgml | 26 + doc/src/sgml/libpq.sgml | 50 + doc/src/sgml/protocol.sgml | 55 + meson.build | 4 +- src/backend/libpq/Makefile | 1 + src/backend/libpq/be-secure-openssl.c | 2 +- src/backend/libpq/compression.c | 127 ++ src/backend/libpq/meson.build | 1 + src/backend/libpq/pqcomm.c | 87 +- src/backend/postmaster/postmaster.c | 10 +- .../libpqwalreceiver/libpqwalreceiver.c | 15 +- src/backend/utils/misc/guc_tables.c | 12 + src/backend/utils/misc/postgresql.conf.sample | 3 + src/bin/pgbench/pgbench.c | 17 +- src/bin/psql/command.c | 18 + src/common/Makefile | 4 +- src/common/compression.c | 145 ++- src/common/io_stream.c | 13 + src/common/meson.build | 2 + src/common/z_stream.c | 995 ++++++++++++++++ src/common/zpq_stream.c | 1035 +++++++++++++++++ src/include/common/compression.h | 8 +- src/include/common/io_stream.h | 17 +- src/include/common/z_stream.h | 98 ++ src/include/common/zpq_stream.h | 91 ++ src/include/libpq/compression.h | 30 + src/include/libpq/libpq-be-fe-helpers.h | 13 +- src/include/libpq/libpq-be.h | 3 + src/include/libpq/libpq.h | 1 + src/include/libpq/protocol.h | 2 +- src/include/utils/guc_hooks.h | 2 + src/interfaces/libpq/exports.txt | 3 + src/interfaces/libpq/fe-connect.c | 129 +- src/interfaces/libpq/fe-exec.c | 15 + src/interfaces/libpq/fe-misc.c | 47 +- src/interfaces/libpq/fe-protocol3.c | 92 +- src/interfaces/libpq/fe-secure-openssl.c | 2 +- src/interfaces/libpq/libpq-fe.h | 4 + src/interfaces/libpq/libpq-int.h | 18 +- 40 files changed, 3076 insertions(+), 147 deletions(-) create mode 100644 src/backend/libpq/compression.c create mode 100644 src/common/z_stream.c create mode 100644 src/common/zpq_stream.c create mode 100644 src/include/common/z_stream.h create mode 100644 src/include/common/zpq_stream.h create mode 100644 src/include/libpq/compression.h diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index 5800c6a9fb..82c7b17b58 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -872,11 +872,14 @@ pgfdw_get_result(PGconn *conn, const char *query) pgfdw_we_get_result = WaitEventExtensionNew("PostgresFdwGetResult"); /* Sleep until there's something to do */ - wc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_SOCKET_READABLE | - WL_EXIT_ON_PM_DEATH, - PQsocket(conn), - -1L, pgfdw_we_get_result); + if (PQreadPending(conn)) + wc = WL_SOCKET_READABLE; + else + wc = WaitLatchOrSocket(MyLatch, + WL_LATCH_SET | WL_SOCKET_READABLE | + WL_EXIT_ON_PM_DEATH, + PQsocket(conn), + -1L, pgfdw_we_get_result); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); @@ -1580,11 +1583,14 @@ pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime, PGresult **result, pgfdw_we_cleanup_result = WaitEventExtensionNew("PostgresFdwCleanupResult"); /* Sleep until there's something to do */ - wc = WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_SOCKET_READABLE | - WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, - PQsocket(conn), - cur_timeout, pgfdw_we_cleanup_result); + if (PQreadPending(conn)) + wc = WL_SOCKET_READABLE; + else + wc = WaitLatchOrSocket(MyLatch, + WL_LATCH_SET | WL_SOCKET_READABLE | + WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + PQsocket(conn), + cur_timeout, pgfdw_we_cleanup_result); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index ee98585027..a86f8e452c 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1066,6 +1066,32 @@ include_dir 'conf.d' </listitem> </varlistentry> + <varlistentry id="guc-libpq-compression" xreflabel="libpq_compression"> + <term><varname>libpq_compression</varname> (<type>string</type>) + <indexterm> + <primary><varname>libpq_compression</varname> configuration parameter</primary> + </indexterm> + </term> + <listitem> + <para> + This parameter controls the available client-server traffic compression methods. + It allows rejecting compression requests even if it is supported by the server (for example, due to security, or CPU consumption). + The default is <literal>off</literal>, which means that no compression methods are allowed. + The value <literal>on</literal> means all supported compression methods are allowed. + For more precise control, a list of the allowed compression methods can be specified. + For example, to allow only <literal>lz4</literal> and <literal>gzip</literal>, set the setting to <literal>lz4;gzip</literal>. + The server will choose the first algorithm from the list also supported by a given client. + <literal>none</literal> is also allowed when specifying a list, and if selected means that the server + will send messages uncompressed, but may still receive compressed messages if the list includes them. + This is most useful if e.g. you want to enable compression for client-to-server traffic + but not client-to-server traffic, using e.g. <literal>none;gzip;lz4</literal> as the + parameter. Also, compression level can be specified for each method, e.g. <literal>lz4:1;gzip:2</literal> + setting will set the compression level for <literal>lz4</literal> to 1 and <literal>gzip</literal> + to 2. The default compression level for each algorithm is chosen by the underlying library. + </para> + </listitem> + </varlistentry> + </variablelist> </sect2> diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index ed88ac001a..b5b9ecc2d6 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -1354,6 +1354,56 @@ postgresql://%2Fvar%2Flib%2Fpostgresql/dbname </listitem> </varlistentry> + <varlistentry id="libpq-connect-compression" xreflabel="compression"> + <term><literal>compression</literal></term> + <listitem> + <para> + Request compression of libpq traffic. The client sends a request with + a list of compression algorithms. Compression can be requested by a client + by including the <literal>"compression"</literal> option in its connection string. + This can either be a boolean value to enable or disable compression + (<literal>"true"</literal>/<literal>"false"</literal>, + <literal>"on"</literal>/<literal>"off"</literal>, + <literal>"yes"</literal>/<literal>"no"</literal>, + <literal>"1"</literal>/<literal>"0"</literal>), + or an explicit list of comma-separated compression algorithms + which can optionally include compression level (<literal>"gzip;lz4:2"</literal>). + The default value is <literal>"off"</literal>. + If compression is enabled but an algorithm is not explicitly specified, + the client library sends its full list of supported algorithms. + The server sends its list of supported parameters in the <varname>libpq_compression</varname> + parameter. + Both the client and server will prefer the first algorithm in their list that is supported by + the other side. + <literal>"none"</literal> is also allowed in the list, and if selected means that the client + (or server) will send messages uncompressed, but depending on the other values it still may + receive compressed messages. + This is most useful if e.g. you want to enable compression for server-to-client traffic + but not client-to-server traffic, using e.g. <literal>"none;gzip;lz4"</literal> as the + compression parameter. + </para> + <para> + After receiving a startup packet with <varname>_pq_.libpq_compression</varname> set, the + server can send CompressedData messages referencing any of the specified algorithms, for + server-to-client traffic compression. + After receiving a parameter status message with <varname>libpq_compression</varname> set, + the client can send CompressedData messages referencing any of the specified algorithms for + client-to-server traffic compression. (Note that if the client has not requested the + <varname>_pq_.libpq_compression</varname> protocol extension in the startup packet, the + server may reject all CompressedData messages even if <varname>libpq_compression</varname> + is non-empty.) + </para> + <para> + Support for compression algorithms must be enabled when the server is compiled. + Currently, three algorithms are supported: gzip (default), lz4 (if Postgres was + configured with --with-lz4 option), and zstd (if Postgres was configured with + --with-zstd option). In all cases, streaming mode is used. + Please note that using compression together with SSL may expose extra vulnerabilities: + <ulink url="https://en.wikipedia.org/wiki/CRIME">CRIME</ulink> + </para> + </listitem> + </varlistentry> + <varlistentry id="libpq-connect-client-encoding" xreflabel="client_encoding"> <term><literal>client_encoding</literal></term> <listitem> diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index af3f016f74..700e125b3c 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -92,6 +92,15 @@ such as <command>COPY</command>. </para> + <para> + The protocol supports compressing data to reduce traffic and speed-up client-server interaction. + Compression is especially useful for importing/exporting data to/from the database using the <literal>COPY</literal> command + and for replication (both physical and logical). Compression can also reduce the server's response time + for queries returning a large amount of data (for example, JSON, BLOBs, text, ...). + Currently, three algorithms are supported: DEFLATE (if PostgreSQL was built with zlib support), + LZ4 (if PostgreSQL was built with lz4 support), and ZStandard (if PostgresSQL was built with zstd support). + </para> + <sect2 id="protocol-message-concepts"> <title>Messaging Overview</title> @@ -4115,6 +4124,52 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" </listitem> </varlistentry> + <varlistentry id="protocol-message-formats-CompressedData"> + <term>CompressedData (F & B)</term> + <listitem> + <para> + + <variablelist> + <varlistentry> + <term>Byte1('z')</term> + <listitem> + <para> + Identifies the message as compressed data. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>Int32</term> + <listitem> + <para> + Length of message contents in bytes, including self. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>Int8</term> + <listitem> + <para> + Selected compression algorithm, as specified in the pg_compress_algorithm enum. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term> + Byte<replaceable>n</replaceable> + </term> + <listitem> + <para> + Compressed message data. + </para> + </listitem> + </varlistentry> + </variablelist> + + </para> + </listitem> + </varlistentry> + <varlistentry id="protocol-message-formats-CopyData"> <term>CopyData (F & B)</term> <listitem> diff --git a/meson.build b/meson.build index f816283301..ffec75a8f8 100644 --- a/meson.build +++ b/meson.build @@ -2793,14 +2793,14 @@ frontend_common_code = declare_dependency( compile_args: ['-DFRONTEND'], include_directories: [postgres_inc], sources: generated_headers, - dependencies: [os_deps, zlib, zstd], + dependencies: [os_deps, lz4, zlib, zstd], ) backend_common_code = declare_dependency( compile_args: ['-DBUILDING_DLL'], include_directories: [postgres_inc], sources: generated_headers, - dependencies: [os_deps, zlib, zstd], + dependencies: [os_deps, lz4, zlib, zstd], ) subdir('src/common') diff --git a/src/backend/libpq/Makefile b/src/backend/libpq/Makefile index 6d385fd6a4..f4b09c09c3 100644 --- a/src/backend/libpq/Makefile +++ b/src/backend/libpq/Makefile @@ -21,6 +21,7 @@ OBJS = \ be-fsstubs.o \ be-secure-common.o \ be-secure.o \ + compression.o \ crypt.o \ hba.o \ ifaddr.o \ diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 5c67fd46aa..0125b50b95 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -78,7 +78,7 @@ static void be_tls_close(Port *port); IoStreamProcessor be_tls_processor = { .read = (io_stream_read_func) be_tls_read, .write = (io_stream_write_func) be_tls_write, - .destroy = (io_stream_destroy_func) be_tls_close + .destroy = (io_stream_consumer) be_tls_close }; static char *X509_NAME_to_cstring(X509_NAME *name); diff --git a/src/backend/libpq/compression.c b/src/backend/libpq/compression.c new file mode 100644 index 0000000000..bb17a23231 --- /dev/null +++ b/src/backend/libpq/compression.c @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------- +* + * compression.c + * Functions and variables to support backend configuration of libpq + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * src/backend/libpq/compression.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "miscadmin.h" +#include "libpq/libpq-be.h" +#include "libpq/compression.h" +#include "utils/guc_hooks.h" + +/* GUC variable containing the allowed compression algorithms list (separated by semicolon) */ +char *libpq_compress_algorithms = "off"; +pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; +size_t libpq_n_compressors = 0; + +bool +check_libpq_compression(char **newval, void **extra, GucSource source) +{ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + char *serialized_compressors; + + if (zpq_parse_compression_setting(*newval, compressors, &n_compressors) == -1) + { + GUC_check_errdetail("Cannot parse the libpq_compression setting."); + return false; + } + + if (n_compressors > 0) + { + guc_free(*newval); + serialized_compressors = zpq_serialize_compressors(compressors, n_compressors); + *newval = guc_strdup(ERROR, serialized_compressors); + pfree(serialized_compressors); + } + else + { + guc_free(*newval); + *newval = guc_strdup(ERROR, ""); + } + return true; +} + +void +assign_libpq_compression(const char *newval, void *extra) +{ + if (strlen(newval) == 0) + { + libpq_n_compressors = 0; + return; + } + zpq_parse_compression_setting(newval, libpq_compressors, &libpq_n_compressors); +} + +void +configure_libpq_compression(Port *port, const char *newval) +{ + pg_compress_specification fe_compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_fe_compressors; + + Assert(!port->zpq_stream); + + if (libpq_n_compressors == 0) + { + return; + } + + /* Init compression */ + port->zpq_stream = zpq_create(libpq_compressors, libpq_n_compressors, MyProcPort->io_stream); + if (!port->zpq_stream) + { + ereport(FATAL, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("failed to initialize the compression stream")); + } + + if (strlen(newval) == 0) + { + return; + } + + if (!zpq_deserialize_compressors(newval, fe_compressors, &n_fe_compressors)) + { + ereport(FATAL, + errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Cannot parse the _pq_.libpq_compression setting.")); + } + + if (MyProcPort && MyProcPort->zpq_stream) + { + pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT]; + size_t n_algorithms = 0; + + if (n_fe_compressors == 0) + { + return; + } + + /* + * Intersect client and server compressors to determine the final list + * of the supported compressors. O(N^2) is negligible because of a + * small number of the compression methods. + */ + for (size_t i = 0; i < libpq_n_compressors; i++) + { + for (size_t j = 0; j < n_fe_compressors; j++) + { + if (libpq_compressors[i].algorithm == fe_compressors[j].algorithm) + { + algorithms[n_algorithms] = libpq_compressors[i].algorithm; + n_algorithms += 1; + break; + } + } + } + + zpq_enable_compression(MyProcPort->zpq_stream, algorithms, n_algorithms); + } +} diff --git a/src/backend/libpq/meson.build b/src/backend/libpq/meson.build index 74a226c2bd..83a21a7566 100644 --- a/src/backend/libpq/meson.build +++ b/src/backend/libpq/meson.build @@ -7,6 +7,7 @@ backend_sources += files( 'be-fsstubs.c', 'be-secure-common.c', 'be-secure.c', + 'compression.c', 'crypt.c', 'hba.c', 'ifaddr.c', diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 030686cc3b..ed4aad57e6 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -61,6 +61,7 @@ #include <signal.h> #include <fcntl.h> #include <grp.h> +#include <pgstat.h> #include <unistd.h> #include <sys/file.h> #include <sys/socket.h> @@ -76,7 +77,9 @@ #include "common/ip.h" #include "common/io_stream.h" +#include "common/zpq_stream.h" #include "libpq/libpq.h" +#include "libpq/pqformat.h" #include "miscadmin.h" #include "port/pg_bswap.h" #include "storage/ipc.h" @@ -84,6 +87,7 @@ #include "utils/guc_hooks.h" #include "utils/memutils.h" #include "utils/wait_event.h" +#include "utils/builtins.h" /* * Cope with the various platform-specific ways to spell TCP keepalive socket @@ -1133,11 +1137,12 @@ retry: /* -------------------------------- * pq_recvbuf - load some bytes into the input buffer * - * returns 0 if OK, EOF if trouble + * nowait parameter toggles non-blocking mode. + * returns number of read bytes, EOF if trouble * -------------------------------- */ static int -pq_recvbuf(void) +pq_recvbuf(bool nowait) { if (PqRecvPointer > 0) { @@ -1153,8 +1158,8 @@ pq_recvbuf(void) PqRecvLength = PqRecvPointer = 0; } - /* Ensure that we're in blocking mode */ - socket_set_nonblocking(false); + /* Ensure that we're in the appropriate mode */ + socket_set_nonblocking(nowait); /* Can fill buffer from PqRecvLength and upwards */ for (;;) @@ -1168,9 +1173,23 @@ pq_recvbuf(void) if (r < 0) { + if (r == ZS_DECOMPRESS_ERROR) + { + char const *msg = zpq_decompress_error(MyProcPort->zpq_stream); + + if (msg == NULL) + msg = "end of stream"; + ereport(COMMERROR, + (errcode_for_socket_access(), + errmsg("failed to decompress data: %s", msg))); + return EOF; + } if (errno == EINTR) continue; /* Ok if interrupted */ + if (nowait && (errno == EAGAIN || errno == EWOULDBLOCK)) + return 0; + /* * Careful: an ereport() that tries to write to the client would * cause recursion to here, leading to stack overflow and core @@ -1194,7 +1213,7 @@ pq_recvbuf(void) } /* r contains number of bytes read, so just incr length */ PqRecvLength += r; - return 0; + return r; } } @@ -1209,7 +1228,8 @@ pq_getbyte(void) while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } return (unsigned char) PqRecvBuffer[PqRecvPointer++]; @@ -1228,7 +1248,8 @@ pq_peekbyte(void) while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } return (unsigned char) PqRecvBuffer[PqRecvPointer]; @@ -1249,49 +1270,12 @@ pq_getbyte_if_available(unsigned char *c) Assert(PqCommReadingMsg); - if (PqRecvPointer < PqRecvLength) + if (PqRecvPointer < PqRecvLength || (r = pq_recvbuf(true)) > 0) { *c = PqRecvBuffer[PqRecvPointer++]; return 1; } - /* Put the socket into non-blocking mode */ - socket_set_nonblocking(true); - - errno = 0; - - r = io_read_with_wait(MyProcPort, c, 1); - if (r < 0) - { - /* - * Ok if no data available without blocking or interrupted (though - * EINTR really shouldn't happen with a non-blocking socket). Report - * other errors. - */ - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) - r = 0; - else - { - /* - * Careful: an ereport() that tries to write to the client would - * cause recursion to here, leading to stack overflow and core - * dump! This message must go *only* to the postmaster log. - * - * If errno is zero, assume it's EOF and let the caller complain. - */ - if (errno != 0) - ereport(COMMERROR, - (errcode_for_socket_access(), - errmsg("could not receive data from client: %m"))); - r = EOF; - } - } - else if (r == 0) - { - /* EOF detected */ - r = EOF; - } - return r; } @@ -1312,7 +1296,8 @@ pq_getbytes(char *s, size_t len) { while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } amount = PqRecvLength - PqRecvPointer; @@ -1346,7 +1331,8 @@ pq_discardbytes(size_t len) { while (PqRecvPointer >= PqRecvLength) { - if (pq_recvbuf()) /* If nothing in buffer, then recv some */ + if (pq_recvbuf(false) == EOF) /* If nothing in buffer, then recv + * some */ return EOF; /* Failed to recv data */ } amount = PqRecvLength - PqRecvPointer; @@ -1574,7 +1560,7 @@ internal_flush(void) char *bufptr = PqSendBuffer + PqSendStart; char *bufend = PqSendBuffer + PqSendPointer; - while (bufptr < bufend) + while (bufptr < bufend || io_stream_buffered_write_data(MyProcPort->io_stream) != 0) { int rc; size_t bytes_sent; @@ -1625,6 +1611,7 @@ internal_flush(void) PqSendStart = PqSendPointer = 0; ClientConnectionLost = 1; InterruptPending = 1; + io_stream_reset_write_state(MyProcPort->io_stream); return EOF; } @@ -1647,7 +1634,7 @@ socket_flush_if_writable(void) int res; /* Quick exit if nothing to do */ - if (PqSendPointer == PqSendStart) + if ((PqSendPointer == PqSendStart) && (io_stream_buffered_write_data(MyProcPort->io_stream) == 0)) return 0; /* No-op if reentrant call */ @@ -1670,7 +1657,7 @@ socket_flush_if_writable(void) static bool socket_is_send_pending(void) { - return (PqSendStart < PqSendPointer); + return (PqSendStart < PqSendPointer || (io_stream_buffered_write_data(MyProcPort->io_stream) != 0)); } /* -------------------------------- diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 1f94e2910c..d83caf20b0 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -99,6 +99,7 @@ #include "common/string.h" #include "lib/ilist.h" #include "libpq/auth.h" +#include "libpq/compression.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" #include "libpq/pqsignal.h" @@ -2210,12 +2211,15 @@ retry1: valptr), errhint("Valid values are: \"false\", 0, \"true\", 1, \"database\"."))); } + else if (strcmp(nameptr, "_pq_.libpq_compression") == 0) + { + configure_libpq_compression(port, valptr); + } else if (strncmp(nameptr, "_pq_.", 5) == 0) { /* * Any option beginning with _pq_. is reserved for use as a - * protocol-level option, but at present no such options are - * defined. + * protocol-level option. */ unrecognized_protocol_options = lappend(unrecognized_protocol_options, pstrdup(nameptr)); @@ -4419,7 +4423,9 @@ BackendInitialize(Port *port) * already did any appropriate error reporting. */ if (status != STATUS_OK) + { proc_exit(0); + } /* * Now that we have the user and database name, we can set the process diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c index 693b3669ba..52e8772ddb 100644 --- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c +++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c @@ -727,12 +727,15 @@ libpqrcv_PQgetResult(PGconn *streamConn) * since we'll get interrupted by signals and can handle any * interrupts here. */ - rc = WaitLatchOrSocket(MyLatch, - WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE | - WL_LATCH_SET, - PQsocket(streamConn), - 0, - WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE); + if (PQreadPending(streamConn)) + rc = WL_SOCKET_READABLE; + else + rc = WaitLatchOrSocket(MyLatch, + WL_EXIT_ON_PM_DEATH | WL_SOCKET_READABLE | + WL_LATCH_SET, + PQsocket(streamConn), + 0, + WAIT_EVENT_LIBPQWALRECEIVER_RECEIVE); /* Interrupted? */ if (rc & WL_LATCH_SET) diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 9f59440526..0d36ca52f2 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -46,6 +46,7 @@ #include "common/scram-common.h" #include "jit/jit.h" #include "libpq/auth.h" +#include "libpq/compression.h" #include "libpq/libpq.h" #include "libpq/scram.h" #include "nodes/queryjumble.h" @@ -4531,6 +4532,17 @@ struct config_string ConfigureNamesString[] = NULL, NULL, NULL }, + { + {"libpq_compression", PGC_SIGHUP, CLIENT_CONN_OTHER, + gettext_noop("Sets the list of allowed libpq compression algorithms."), + NULL, + GUC_REPORT + }, + &libpq_compress_algorithms, + "off", + check_libpq_compression, assign_libpq_compression, NULL + }, + { {"application_name", PGC_USERSET, LOGGING_WHAT, gettext_noop("Sets the application name to be reported in statistics and logs."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b2809c711a..0411571c02 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -74,6 +74,9 @@ # (change requires restart) #bonjour_name = '' # defaults to the computer name # (change requires restart) +#libpq_compression = off # on to allow all supported compression + # methods; off to disable all compression; + # semicolon separated list of algorithms to allow some # - TCP settings - # see "man tcp" for details diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c index 2e1650d0ad..09d43cf960 100644 --- a/src/bin/pgbench/pgbench.c +++ b/src/bin/pgbench/pgbench.c @@ -7433,6 +7433,9 @@ threadRun(void *arg) int nsocks; /* number of sockets to be waited for */ pg_time_usec_t min_usec; pg_time_usec_t now = 0; /* set this only if needed */ + bool buffered_rx = false; /* true if some of the clients has + * data left in SSL/ZPQ read + * buffers */ /* * identify which client sockets should be checked for input, and @@ -7468,6 +7471,9 @@ threadRun(void *arg) */ int sock = PQsocket(st->con); + /* check if conn has buffered SSL / ZPQ read data */ + buffered_rx = buffered_rx || PQreadPending(st->con); + if (sock < 0) { pg_log_error("invalid socket: %s", PQerrorMessage(st->con)); @@ -7511,7 +7517,7 @@ threadRun(void *arg) { if (nsocks > 0) { - rc = wait_on_socket_set(sockets, min_usec); + rc = buffered_rx ? 1 : wait_on_socket_set(sockets, min_usec); } else /* nothing active, simple sleep */ { @@ -7520,7 +7526,7 @@ threadRun(void *arg) } else /* no explicit delay, wait without timeout */ { - rc = wait_on_socket_set(sockets, 0); + rc = buffered_rx ? 1 : wait_on_socket_set(sockets, 0); } if (rc < 0) @@ -7560,8 +7566,11 @@ threadRun(void *arg) pg_log_error("invalid socket: %s", PQerrorMessage(st->con)); goto done; } - - if (!socket_has_input(sockets, sock, nsocks++)) + if (PQreadPending(st->con)) + { + nsocks++; + } + else if (!socket_has_input(sockets, sock, nsocks++)) continue; } else if (st->state == CSTATE_FINISHED || diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c index 82cc091568..4ed6fb5d21 100644 --- a/src/bin/psql/command.c +++ b/src/bin/psql/command.c @@ -172,6 +172,7 @@ static int count_lines_in_buf(PQExpBuffer buf); static void print_with_linenumbers(FILE *output, char *lines, bool is_func); static void minimal_error_message(PGresult *res); +static void printCompressionInfo(void); static void printSSLInfo(void); static void printGSSInfo(void); static bool printPsetInfo(const char *param, printQueryOpt *popt); @@ -676,6 +677,7 @@ exec_command_conninfo(PsqlScanState scan_state, bool active_branch) printf(_("You are connected to database \"%s\" as user \"%s\" on host \"%s\" at port \"%s\".\n"), db, PQuser(pset.db), host, PQport(pset.db)); } + printCompressionInfo(); printSSLInfo(); printGSSInfo(); } @@ -3821,6 +3823,22 @@ connection_warnings(bool in_startup) } } +/* + * printCompressionInfo + * + * Print information about used compressor/decompressor + */ +static void +printCompressionInfo(void) +{ + char *algorithms = PQcompression(pset.db); + + if (algorithms != NULL) + { + printf(_("Compression: server: %s, client: %s\n"), PQserverCompression(pset.db), algorithms); + pfree(algorithms); + } +} /* * printSSLInfo diff --git a/src/common/Makefile b/src/common/Makefile index a80f4b39a6..49dc539e18 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -83,7 +83,9 @@ OBJS_COMMON = \ unicode_norm.o \ username.o \ wait_error.o \ - wchar.o + wchar.o \ + z_stream.o \ + zpq_stream.o ifeq ($(with_ssl),openssl) OBJS_COMMON += \ diff --git a/src/common/compression.c b/src/common/compression.c index ee937623f0..c2b00aaf11 100644 --- a/src/common/compression.c +++ b/src/common/compression.c @@ -36,6 +36,16 @@ #include "common/compression.h" +#ifndef FRONTEND +#define ALLOC palloc +#define STRDUP pstrdup +#define FREE pfree +#else +#define ALLOC malloc +#define STRDUP strdup +#define FREE free +#endif + static int expect_integer_value(char *keyword, char *value, pg_compress_specification *result); static bool expect_boolean_value(char *keyword, char *value, @@ -113,7 +123,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* Initial setup of result object. */ result->algorithm = algorithm; result->options = 0; - result->parse_error = NULL; + result->has_error = false; /* * Assign a default level depending on the compression method. This may @@ -128,27 +138,27 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio #ifdef USE_LZ4 result->level = 0; /* fast compression mode */ #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "LZ4"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "LZ4"); #endif break; case PG_COMPRESSION_ZSTD: #ifdef USE_ZSTD result->level = ZSTD_CLEVEL_DEFAULT; #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "ZSTD"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "ZSTD"); #endif break; case PG_COMPRESSION_GZIP: #ifdef HAVE_LIBZ result->level = Z_DEFAULT_COMPRESSION; #else - result->parse_error = - psprintf(_("this build does not support compression with %s"), - "gzip"); + result->has_error = true; + snprintf(result->parse_error, 255, _("this build does not support compression with %s"), + "gzip"); #endif break; } @@ -201,20 +211,20 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* Reject empty keyword. */ if (kwlen == 0) { - result->parse_error = - pstrdup(_("found empty string where a compression option was expected")); + result->has_error = true; + strlcpy(result->parse_error, _("found empty string where a compression option was expected"), 255); break; } /* Extract keyword and value as separate C strings. */ - keyword = palloc(kwlen + 1); + keyword = ALLOC(kwlen + 1); memcpy(keyword, kwstart, kwlen); keyword[kwlen] = '\0'; if (!has_value) value = NULL; else { - value = palloc(vlen + 1); + value = ALLOC(vlen + 1); memcpy(value, vstart, vlen); value[vlen] = '\0'; } @@ -240,13 +250,15 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio result->options |= PG_COMPRESSION_OPTION_LONG_DISTANCE; } else - result->parse_error = - psprintf(_("unrecognized compression option: \"%s\""), keyword); + { + result->has_error = true; + snprintf(result->parse_error, 255, _("unrecognized compression option: \"%s\""), keyword); + } /* Release memory, just to be tidy. */ - pfree(keyword); + FREE(keyword); if (value != NULL) - pfree(value); + FREE(value); /* * If we got an error or have reached the end of the string, stop. @@ -256,7 +268,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio * keyword cannot have been the end of the string, but the end of the * value might have been. */ - if (result->parse_error != NULL || + if (result->has_error || (vend == NULL ? *kwend == '\0' : *vend == '\0')) break; @@ -268,7 +280,7 @@ parse_compress_specification(pg_compress_algorithm algorithm, char *specificatio /* * Parse 'value' as an integer and return the result. * - * If parsing fails, set result->parse_error to an appropriate message + * If parsing fails, set result->has_error and write an appropriate message to result->parse_error * and return -1. */ static int @@ -279,18 +291,17 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu if (value == NULL) { - result->parse_error = - psprintf(_("compression option \"%s\" requires a value"), - keyword); + result->has_error = true; + snprintf(result->parse_error, 255, _("compression option \"%s\" requires a value"), keyword); return -1; } ivalue = strtol(value, &ivalue_endp, 10); if (ivalue_endp == value || *ivalue_endp != '\0') { - result->parse_error = - psprintf(_("value for compression option \"%s\" must be an integer"), - keyword); + result->has_error = true; + snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be an integer"), + keyword); return -1; } return ivalue; @@ -299,8 +310,8 @@ expect_integer_value(char *keyword, char *value, pg_compress_specification *resu /* * Parse 'value' as a boolean and return the result. * - * If parsing fails, set result->parse_error to an appropriate message - * and return -1. The caller must check result->parse_error to determine if + * If parsing fails, set result->has_error and write an appropriate message to result->parse_error + * and return -1. The caller must check result->has_error to determine if * the call was successful. * * Valid values are: yes, no, on, off, 1, 0. @@ -327,9 +338,10 @@ expect_boolean_value(char *keyword, char *value, pg_compress_specification *resu if (pg_strcasecmp(value, "0") == 0) return false; - result->parse_error = - psprintf(_("value for compression option \"%s\" must be a Boolean value"), - keyword); + + result->has_error = true; + snprintf(result->parse_error, 255, _("value for compression option \"%s\" must be a Boolean value"), + keyword); return false; } @@ -348,7 +360,7 @@ validate_compress_specification(pg_compress_specification *spec) int default_level = 0; /* If it didn't even parse OK, it's definitely no good. */ - if (spec->parse_error != NULL) + if (spec->has_error) return spec->parse_error; /* @@ -376,16 +388,22 @@ validate_compress_specification(pg_compress_specification *spec) break; case PG_COMPRESSION_NONE: if (spec->level != 0) - return psprintf(_("compression algorithm \"%s\" does not accept a compression level"), - get_compress_algorithm_name(spec->algorithm)); + { + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a compression level"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; + } break; } if ((spec->level < min_level || spec->level > max_level) && spec->level != default_level) - return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"), - get_compress_algorithm_name(spec->algorithm), - min_level, max_level, default_level); + { + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"), + get_compress_algorithm_name(spec->algorithm), + min_level, max_level, default_level); + return spec->parse_error; + } /* * Of the compression algorithms that we currently support, only zstd @@ -394,8 +412,9 @@ validate_compress_specification(pg_compress_specification *spec) if ((spec->options & PG_COMPRESSION_OPTION_WORKERS) != 0 && (spec->algorithm != PG_COMPRESSION_ZSTD)) { - return psprintf(_("compression algorithm \"%s\" does not accept a worker count"), - get_compress_algorithm_name(spec->algorithm)); + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not accept a worker count"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; } /* @@ -405,13 +424,45 @@ validate_compress_specification(pg_compress_specification *spec) if ((spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) != 0 && (spec->algorithm != PG_COMPRESSION_ZSTD)) { - return psprintf(_("compression algorithm \"%s\" does not support long-distance mode"), - get_compress_algorithm_name(spec->algorithm)); + snprintf(spec->parse_error, 255, _("compression algorithm \"%s\" does not support long-distance mode"), + get_compress_algorithm_name(spec->algorithm)); + return spec->parse_error; } return NULL; } +bool +supported_compression_algorithm(pg_compress_algorithm algorithm) +{ + switch (algorithm) + { + case PG_COMPRESSION_NONE: + return true; + case PG_COMPRESSION_GZIP: +#ifdef HAVE_LIBZ + return true; +#else + return false; +#endif + case PG_COMPRESSION_LZ4: +#ifdef USE_LZ4 + return true; +#else + return false; +#endif + case PG_COMPRESSION_ZSTD: +#ifdef USE_ZSTD + return true; +#else + return false; +#endif + /* no default, to provoke compiler warnings if values are added */ + } + Assert(false); + return false; /* placate compiler */ +} + #ifdef FRONTEND /* @@ -440,13 +491,13 @@ parse_compress_options(const char *option, char **algorithm, char **detail) { if (result == 0) { - *algorithm = pstrdup("none"); + *algorithm = STRDUP("none"); *detail = NULL; } else { - *algorithm = pstrdup("gzip"); - *detail = pstrdup(option); + *algorithm = STRDUP("gzip"); + *detail = STRDUP(option); } return; } @@ -458,19 +509,19 @@ parse_compress_options(const char *option, char **algorithm, char **detail) sep = strchr(option, ':'); if (sep == NULL) { - *algorithm = pstrdup(option); + *algorithm = STRDUP(option); *detail = NULL; } else { char *alg; - alg = palloc((sep - option) + 1); + alg = ALLOC((sep - option) + 1); memcpy(alg, option, sep - option); alg[sep - option] = '\0'; *algorithm = alg; - *detail = pstrdup(sep + 1); + *detail = STRDUP(sep + 1); } } #endif /* FRONTEND */ diff --git a/src/common/io_stream.c b/src/common/io_stream.c index b15aca326d..6598d12841 100644 --- a/src/common/io_stream.c +++ b/src/common/io_stream.c @@ -129,6 +129,19 @@ io_stream_buffered_write_data(IoStream * stream) return false; } +void +io_stream_reset_write_state(IoStream * stream) +{ + + IoStreamLayer *layer; + + for (layer = stream->layer; layer != NULL; layer = layer->next) + { + if (layer->processor->reset_write_state != NULL) + layer->processor->reset_write_state(layer->context); + } +} + ssize_t io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only) { diff --git a/src/common/meson.build b/src/common/meson.build index e25fa44133..99ef06cd4c 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -38,6 +38,8 @@ common_sources = files( 'username.c', 'wait_error.c', 'wchar.c', + 'z_stream.c', + 'zpq_stream.c' ) if ssl.found() diff --git a/src/common/z_stream.c b/src/common/z_stream.c new file mode 100644 index 0000000000..eb34733702 --- /dev/null +++ b/src/common/z_stream.c @@ -0,0 +1,995 @@ +/*------------------------------------------------------------------------- + * + * z_stream.c + * Functions implementing streaming compression algorithms + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/z_stream.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" +#include "pg_config.h" +#include "common/z_stream.h" +#include "utils/elog.h" + +typedef struct +{ + /* + * Id of compression algorithm. + */ + pg_compress_algorithm algorithm; + + /* + * Create new compression stream. level: compression level + */ + void *(*create_compressor) (int level); + + /* + * Create new decompression stream. + */ + void *(*create_decompressor) (); + + /* + * Decompress up to "src_size" compressed bytes from *src and write up to + * "dst_size" raw (decompressed) bytes to *dst. Number of decompressed + * bytes written to *dst is stored in *dst_processed. Number of compressed + * bytes read from *src is stored in *src_processed. + * + * Return codes: ZS_OK if no errors were encountered during decompression + * attempt. This return code does not guarantee that *src_processed > 0 or + * *dst_processed > 0. + * + * ZS_STREAM_END if encountered end of compressed data stream. + * + * ZS_DECOMPRESS_ERROR if encountered an error during decompression + * attempt. + */ + int (*decompress) (void *ds, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + + /* + * Returns true if there is some data left in internal decompression + * buffers + */ + bool (*decompress_buffered_data) (void *ds); + + /* + * Compress up to "src_size" raw (non-compressed) bytes from *src and + * write up to "dst_size" compressed bytes to *dst. Number of compressed + * bytes written to *dst is stored in *dst_processed. Number of + * non-compressed bytes read from *src is stored in *src_processed. + * + * Return codes: ZS_OK if no errors were encountered during compression + * attempt. This return code does not guarantee that *src_processed > 0 or + * *dst_processed > 0. + * + * ZS_COMPRESS_ERROR if encountered an error during compression attempt. + */ + int (*compress) (void *cs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + + /* + * Returns true if there is some data left in internal compression buffers + */ + bool (*compress_buffered_data) (void *ds); + + /* + * Free compression stream created by create_compressor function. + */ + void (*free_compressor) (void *cs); + + /* + * Free decompression stream created by create_decompressor function. + */ + void (*free_decompressor) (void *ds); + + /* + * Get compressor error message. + */ + char const *(*compress_error) (void *cs); + + /* + * Get decompressor error message. + */ + char const *(*decompress_error) (void *ds); + + int (*end_compression) (void *cs, void *dst, size_t dst_size, size_t *dst_processed); +} ZAlgorithm; + +struct ZStream +{ + ZAlgorithm const *algorithm; + void *stream; +}; + +#ifndef FRONTEND +#include "utils/palloc.h" +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define FREE(size) free(size) +#endif + +#if HAVE_LIBZSTD + +#include <stdlib.h> +#include <zstd.h> + +/* + * Maximum allowed back-reference distance, expressed as power of 2. + * This setting controls max compressor/decompressor window size. + * More details https://github.com/facebook/zstd/blob/v1.4.7/lib/zstd.h#L536 + */ +#define ZSTD_WINDOWLOG_LIMIT 23 /* set max window size to 8MB */ + + +typedef struct ZS_ZSTD_CStream +{ + ZSTD_CStream *stream; + bool has_buffered_data; + char const *error; /* error message */ +} ZS_ZSTD_CStream; + +typedef struct ZS_ZSTD_DStream +{ + ZSTD_DStream *stream; + bool has_buffered_data; + char const *error; /* error message */ +} ZS_ZSTD_DStream; + +static void * +zstd_create_compressor(int level) +{ + size_t rc; + ZS_ZSTD_CStream *c_stream = (ZS_ZSTD_CStream *) ALLOC(sizeof(ZS_ZSTD_CStream)); + + c_stream->stream = ZSTD_createCStream(); + c_stream->has_buffered_data = false; + rc = ZSTD_initCStream(c_stream->stream, level); + if (ZSTD_isError(rc)) + { + ZSTD_freeCStream(c_stream->stream); + FREE(c_stream); + return NULL; + } +#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3 + ZSTD_CCtx_setParameter(c_stream->stream, ZSTD_c_windowLog, ZSTD_WINDOWLOG_LIMIT); +#endif + c_stream->error = NULL; + return c_stream; +} + +static void * +zstd_create_decompressor() +{ + size_t rc; + ZS_ZSTD_DStream *d_stream = (ZS_ZSTD_DStream *) ALLOC(sizeof(ZS_ZSTD_DStream)); + + d_stream->stream = ZSTD_createDStream(); + d_stream->has_buffered_data = false; + rc = ZSTD_initDStream(d_stream->stream); + if (ZSTD_isError(rc)) + { + ZSTD_freeDStream(d_stream->stream); + FREE(d_stream); + return NULL; + } +#if ZSTD_VERSION_MAJOR > 1 || ZSTD_VERSION_MINOR > 3 + ZSTD_DCtx_setParameter(d_stream->stream, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT); +#endif + d_stream->error = NULL; + return d_stream; +} + +static int +zstd_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + size_t rc; + + in.src = src; + in.pos = 0; + in.size = src_size; + + out.dst = dst; + out.pos = 0; + out.size = dst_size; + + rc = ZSTD_decompressStream(ds->stream, &out, &in); + + *src_processed = in.pos; + *dst_processed = out.pos; + if (ZSTD_isError(rc)) + { + ds->error = ZSTD_getErrorName(rc); + return ZS_DECOMPRESS_ERROR; + } + + if (rc == 0) + { + return ZS_STREAM_END; + } + + /* + * if `output.pos == output.size`, there might be some data left within + * internal buffers + */ + ds->has_buffered_data = out.pos == out.size; + + return ZS_OK; +} + +static bool +zstd_decompress_buffered_data(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + return ds->has_buffered_data; +} + +static int +zstd_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + + in.src = src; + in.pos = 0; + in.size = src_size; + + out.dst = dst; + out.pos = 0; + out.size = dst_size; + + if (in.pos < src_size) /* Has something to compress in input buffer */ + { + size_t rc = ZSTD_compressStream(cs->stream, &out, &in); + + *dst_processed = out.pos; + *src_processed = in.pos; + if (ZSTD_isError(rc)) + { + cs->error = ZSTD_getErrorName(rc); + return ZS_COMPRESS_ERROR; + } + } + + if (in.pos == src_size) /* All data is compressed: flush internal zstd + * buffer */ + { + size_t tx_not_flushed = ZSTD_flushStream(cs->stream, &out); + + *dst_processed = out.pos; + cs->has_buffered_data = tx_not_flushed > 0; + } + else + { + cs->has_buffered_data = false; + } + + return ZS_OK; +} + +static bool +zstd_compress_buffered_data(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + return cs->has_buffered_data; +} + +static int +zstd_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + size_t tx_not_flushed; + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + ZSTD_outBuffer output; + + output.dst = dst; + output.pos = 0; + output.size = dst_size; + + do + { + tx_not_flushed = ZSTD_endStream(cs->stream, &output); + } while ((tx_not_flushed > 0) && (output.pos < output.size)); + + *dst_processed = output.pos; + + cs->has_buffered_data = tx_not_flushed > 0; + return ZS_OK; +} + +static void +zstd_free_compressor(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + if (cs != NULL) + { + ZSTD_freeCStream(cs->stream); + FREE(cs); + } +} + +static void +zstd_free_decompressor(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + if (ds != NULL) + { + ZSTD_freeDStream(ds->stream); + FREE(ds); + } +} + +static char const * +zstd_compress_error(void *c_stream) +{ + ZS_ZSTD_CStream *cs = (ZS_ZSTD_CStream *) c_stream; + + return cs->error; +} + +static char const * +zstd_decompress_error(void *d_stream) +{ + ZS_ZSTD_DStream *ds = (ZS_ZSTD_DStream *) d_stream; + + return ds->error; +} + +static ZAlgorithm const zstd_algorithm = { + .algorithm = PG_COMPRESSION_ZSTD, + .create_compressor = zstd_create_compressor, + .create_decompressor = zstd_create_decompressor, + .decompress = zstd_decompress, + .decompress_buffered_data = zstd_decompress_buffered_data, + .compress = zstd_compress, + .compress_buffered_data = zstd_compress_buffered_data, + .free_compressor = zstd_free_compressor, + .free_decompressor = zstd_free_decompressor, + .compress_error = zstd_compress_error, + .decompress_error = zstd_decompress_error, + .end_compression = zstd_end +}; + +#endif + +#if HAVE_LIBZ + +#include <stdlib.h> +#include <zlib.h> + + +static void * +zlib_create_compressor(int level) +{ + int rc; + z_stream *c_stream = (z_stream *) ALLOC(sizeof(z_stream)); + + memset(c_stream, 0, sizeof(*c_stream)); + rc = deflateInit(c_stream, level); + if (rc != Z_OK) + { + FREE(c_stream); + return NULL; + } + return c_stream; +} + +static void * +zlib_create_decompressor() +{ + int rc; + z_stream *d_stream = (z_stream *) ALLOC(sizeof(z_stream)); + + memset(d_stream, 0, sizeof(*d_stream)); + rc = inflateInit(d_stream); + if (rc != Z_OK) + { + FREE(d_stream); + return NULL; + } + return d_stream; +} + +static int +zlib_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *ds = (z_stream *) d_stream; + int rc; + + ds->next_in = (Bytef *) src; + ds->avail_in = src_size; + ds->next_out = (Bytef *) dst; + ds->avail_out = dst_size; + + rc = inflate(ds, Z_SYNC_FLUSH); + *src_processed = src_size - ds->avail_in; + *dst_processed = dst_size - ds->avail_out; + + if (rc == Z_STREAM_END) + { + return ZS_STREAM_END; + } + if (rc != Z_OK && rc != Z_BUF_ERROR) + { + return ZS_DECOMPRESS_ERROR; + } + + return ZS_OK; +} + +static bool +zlib_decompress_buffered_data(void *d_stream) +{ + z_stream *ds = (z_stream *) d_stream; + unsigned deflate_pending = 0; + + return deflatePending(ds, &deflate_pending, Z_NULL) > 0; +} + +static int +zlib_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *cs = (z_stream *) c_stream; + int rc PG_USED_FOR_ASSERTS_ONLY; + + cs->next_out = (Bytef *) dst; + cs->avail_out = dst_size; + cs->next_in = (Bytef *) src; + cs->avail_in = src_size; + + rc = deflate(cs, Z_SYNC_FLUSH); + Assert(rc == Z_OK); + *dst_processed = dst_size - cs->avail_out; + *src_processed = src_size - cs->avail_in; + + return ZS_OK; +} + +static bool +zlib_compress_buffered_data(void *c_stream) +{ + return false; +} + +static int +zlib_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + z_stream *cs = (z_stream *) c_stream; + int rc; + + cs->next_out = (Bytef *) dst; + cs->avail_out = dst_size; + cs->next_in = NULL; + cs->avail_in = 0; + + rc = deflate(cs, Z_STREAM_END); + Assert(rc == Z_OK || rc == Z_STREAM_END); + *dst_processed = dst_size - cs->avail_out; + if (rc == Z_STREAM_END) + { + return ZS_OK; + } + + return rc; +} + +static void +zlib_free_compressor(void *c_stream) +{ + z_stream *cs = (z_stream *) c_stream; + + if (cs != NULL) + { + deflateEnd(cs); + FREE(cs); + } +} + +static void +zlib_free_decompressor(void *d_stream) +{ + z_stream *ds = (z_stream *) d_stream; + + if (ds != NULL) + { + inflateEnd(ds); + FREE(ds); + } +} + +static char const * +zlib_error(void *stream) +{ + z_stream *zs = (z_stream *) stream; + + return zs->msg; +} + +/* as with elsewhere in postgres, gzip really means zlib */ +static ZAlgorithm const zlib_algorithm = { + .algorithm = PG_COMPRESSION_GZIP, + .create_compressor = zlib_create_compressor, + .create_decompressor = zlib_create_decompressor, + .decompress = zlib_decompress, + .decompress_buffered_data = zlib_decompress_buffered_data, + .compress = zlib_compress, + .compress_buffered_data = zlib_compress_buffered_data, + .free_compressor = zlib_free_compressor, + .free_decompressor = zlib_free_decompressor, + .compress_error = zlib_error, + .decompress_error = zlib_error, + .end_compression = zlib_end +}; + +#endif + +#if USE_LZ4 +#include <lz4.h> + +#define MESSAGE_MAX_BYTES 64 * 1024 +#define RING_BUFFER_BYTES (LZ4_DECODER_RING_BUFFER_SIZE(MESSAGE_MAX_BYTES)) + +typedef struct ZS_LZ4_CStream +{ + LZ4_stream_t *stream; + int level; + size_t buf_pos; + char *last_error; + char buf[RING_BUFFER_BYTES]; +} ZS_LZ4_CStream; + +typedef struct ZS_LZ4_DStream +{ + LZ4_streamDecode_t *stream; + size_t buf_pos; + size_t read_pos; + char *last_error; + char buf[RING_BUFFER_BYTES]; +} ZS_LZ4_DStream; + +static void * +lz4_create_compressor(int level) +{ + ZS_LZ4_CStream *c_stream = (ZS_LZ4_CStream *) ALLOC(sizeof(ZS_LZ4_CStream)); + + if (c_stream == NULL) + { + return NULL; + } + c_stream->stream = LZ4_createStream(); + c_stream->level = level; + c_stream->buf_pos = 0; + if (c_stream->stream == NULL) + { + FREE(c_stream); + return NULL; + } + return c_stream; +} + +static void * +lz4_create_decompressor() +{ + ZS_LZ4_DStream *d_stream = (ZS_LZ4_DStream *) ALLOC(sizeof(ZS_LZ4_DStream)); + + if (d_stream == NULL) + { + return NULL; + } + + d_stream->stream = LZ4_createStreamDecode(); + d_stream->buf_pos = 0; + d_stream->read_pos = 0; + if (d_stream->stream == NULL) + { + FREE(d_stream); + return NULL; + } + + return d_stream; +} +char last_error[256]; + +static int +lz4_decompress(void *d_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + size_t copyable; + char *decPtr; + int decBytes; + + if (ds->read_pos < ds->buf_pos) + { + decPtr = &ds->buf[ds->read_pos]; + copyable = Min(ds->buf_pos - ds->read_pos, dst_size); + memcpy(dst, decPtr, copyable); /* read msg length */ + *dst_processed = copyable; + *src_processed = 0; + ds->read_pos += copyable; + + if (ds->read_pos == ds->buf_pos && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES) + { + ds->buf_pos = 0; + ds->read_pos = 0; + } + return ZS_OK; + } + decPtr = &ds->buf[ds->buf_pos]; + + decBytes = LZ4_decompress_safe_continue(ds->stream, src, decPtr, (int) src_size, RING_BUFFER_BYTES - ds->buf_pos); + if (decBytes < 0) + { + sprintf(last_error, "LZ4 decompression failed (src_size %ld, dst_size %ld, error: %d)", src_size, RING_BUFFER_BYTES - ds->buf_pos, decBytes); + ds->last_error = last_error; +#ifndef FRONTEND + elog(ERROR, "%s", ds->last_error); +#else + return ZS_DECOMPRESS_ERROR; +#endif + } + + copyable = Min(decBytes, dst_size); + memcpy(dst, decPtr, copyable); + + *dst_processed = copyable; + *src_processed = src_size; + + ds->buf_pos += decBytes; + ds->read_pos += copyable; + /* only reset the ring buffer after the internal buffer is drained */ + if (copyable == decBytes && RING_BUFFER_BYTES - ds->buf_pos < MESSAGE_MAX_BYTES) + { + ds->buf_pos = 0; + ds->read_pos = 0; + } + + return ZS_OK; +} + +static bool +lz4_decompress_buffered_data(void *d_stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + + return ds->read_pos < ds->buf_pos; +} + +static int +lz4_compress(void *c_stream, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream; + int cmpBytes; + + src_size = Min(MESSAGE_MAX_BYTES, src_size); + + memcpy((char *) (cs->buf) + cs->buf_pos, src, src_size); /* write msg length */ + + if (dst_size < LZ4_compressBound(src_size)) + { + cs->last_error = "LZ4 compression failed: buffer not big enough"; +#ifndef FRONTEND + elog(ERROR, "%s", cs->last_error); +#else + return ZS_COMPRESS_ERROR; +#endif + } + + cmpBytes = LZ4_compress_fast_continue(cs->stream, (char *) (cs->buf) + cs->buf_pos, dst, (int) src_size, (int) dst_size, cs->level); + + if (cmpBytes < 0 || cmpBytes > MESSAGE_MAX_BYTES) + { + cs->last_error = "LZ4 compression failed"; +#ifndef FRONTEND + elog(ERROR, "%s", cs->last_error); +#else + return ZS_COMPRESS_ERROR; +#endif + } + + *dst_processed = cmpBytes; + *src_processed = src_size; + + cs->buf_pos += src_size; + if (cs->buf_pos >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) + { + cs->buf_pos = 0; + } + return ZS_OK; +} + +static bool +lz4_compress_buffered_data(void *d_stream) +{ + return false; +} + + +static int +lz4_end(void *c_stream, void *dst, size_t dst_size, size_t *dst_processed) +{ + *dst_processed = 0; + return ZS_OK; +} + +static void +lz4_free_compressor(void *c_stream) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) c_stream; + + if (cs != NULL) + { + if (cs->stream != NULL) + { + LZ4_freeStream(cs->stream); + } + FREE(cs); + } +} + +static void +lz4_free_decompressor(void *d_stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) d_stream; + + if (ds != NULL) + { + if (ds->stream != NULL) + { + LZ4_freeStreamDecode(ds->stream); + } + FREE(ds); + } +} + +static char const * +lz4_compress_error(void *stream) +{ + ZS_LZ4_CStream *cs = (ZS_LZ4_CStream *) stream; + + /* lz4 doesn't have any explicit API to get the error names */ + return cs->last_error; +} + +static char const * +lz4_decompress_error(void *stream) +{ + ZS_LZ4_DStream *ds = (ZS_LZ4_DStream *) stream; + + /* lz4 doesn't have any explicit API to get the error names */ + return ds->last_error; +} + +static ZAlgorithm const lz4_algorithm = { + .algorithm = PG_COMPRESSION_LZ4, + .create_compressor = lz4_create_compressor, + .create_decompressor = lz4_create_decompressor, + .decompress = lz4_decompress, + .decompress_buffered_data = lz4_decompress_buffered_data, + .compress = lz4_compress, + .compress_buffered_data = lz4_compress_buffered_data, + .free_compressor = lz4_free_compressor, + .free_decompressor = lz4_free_decompressor, + .compress_error = lz4_compress_error, + .decompress_error = lz4_decompress_error, + .end_compression = lz4_end +}; + +#endif + +static const ZAlgorithm * +zs_find_algorithm(pg_compress_algorithm algorithm) +{ +#if HAVE_LIBZ + if (algorithm == PG_COMPRESSION_GZIP) + { + return &zlib_algorithm; + } +#endif +#if USE_LZ4 + if (algorithm == PG_COMPRESSION_LZ4) + { + return &lz4_algorithm; + } +#endif +#if HAVE_LIBZSTD + if (algorithm == PG_COMPRESSION_ZSTD) + { + return &zstd_algorithm; + } +#endif + return NULL; +} + +static int +zs_init_compressor(ZStream * zs, pg_compress_specification *spec) +{ + const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm); + + if (algorithm == NULL) + { + return -1; + } + zs->algorithm = algorithm; + zs->stream = zs->algorithm->create_compressor(spec->level); + if (zs->stream == NULL) + { + return -1; + } + return 0; +} + +static int +zs_init_decompressor(ZStream * zs, pg_compress_specification *spec) +{ + const ZAlgorithm *algorithm = zs_find_algorithm(spec->algorithm); + + if (algorithm == NULL) + { + return -1; + } + zs->algorithm = algorithm; + zs->stream = zs->algorithm->create_decompressor(); + if (zs->stream == NULL) + { + return -1; + } + return 0; +} + +ZStream * +zs_create_compressor(pg_compress_specification *spec) +{ + ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream)); + + if (zs_init_compressor(zs, spec)) + { + FREE(zs); + return NULL; + } + + return zs; +} + +ZStream * +zs_create_decompressor(pg_compress_specification *spec) +{ + ZStream *zs = (ZStream *) ALLOC(sizeof(ZStream)); + + if (zs_init_decompressor(zs, spec)) + { + FREE(zs); + return NULL; + } + + return zs; +} + +int +zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *src_processed = 0; + *dst_processed = 0; + + rc = zs->algorithm->decompress(zs->stream, + src, src_size, src_processed, + dst, dst_size, dst_processed); + + if (rc == ZS_OK || rc == ZS_INCOMPLETE_SRC || rc == ZS_STREAM_END) + { + return rc; + } + + return ZS_DECOMPRESS_ERROR; +} + +int +zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *processed = 0; + *dst_processed = 0; + + rc = zs->algorithm->compress(zs->stream, + buf, size, processed, + dst, dst_size, dst_processed); + + if (rc != ZS_OK) + { + return ZS_COMPRESS_ERROR; + } + + return rc; +} + +void +zs_compressor_free(ZStream * zs) +{ + if (zs == NULL) + { + return; + } + + if (zs->stream) + { + zs->algorithm->free_compressor(zs->stream); + } + + FREE(zs); +} + +void +zs_decompressor_free(ZStream * zs) +{ + if (zs == NULL) + { + return; + } + + if (zs->stream) + { + zs->algorithm->free_decompressor(zs->stream); + } + + FREE(zs); +} + +int +zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed) +{ + int rc; + + *dst_processed = 0; + + rc = zs->algorithm->end_compression(zs->stream, dst, dst_size, dst_processed); + + if (rc != ZS_OK) + { + return ZS_COMPRESS_ERROR; + } + + return rc; +} + +char const * +zs_compress_error(ZStream * zs) +{ + return zs->algorithm->compress_error(zs->stream); +} + +char const * +zs_decompress_error(ZStream * zs) +{ + return zs->algorithm->decompress_error(zs->stream); +} + +bool +zs_compress_buffered(ZStream * zs) +{ + return zs ? zs->algorithm->compress_buffered_data(zs->stream) : false; +} + +bool +zs_decompress_buffered(ZStream * zs) +{ + return zs ? zs->algorithm->decompress_buffered_data(zs->stream) : false; +} + +pg_compress_algorithm +zs_algorithm(ZStream * zs) +{ + return zs ? zs->algorithm->algorithm : PG_COMPRESSION_NONE; +} diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c new file mode 100644 index 0000000000..b2a47c7537 --- /dev/null +++ b/src/common/zpq_stream.c @@ -0,0 +1,1035 @@ +/*------------------------------------------------------------------------- + * + * zpq_stream.c + * IO stream layer applying ZStream compression to libpq + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/zpq_stream.c + * + *------------------------------------------------------------------------- + */ +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif +#include <unistd.h> +#include <math.h> + +#include "common/zpq_stream.h" + +#include <common/io_stream.h> +#include <libpq/protocol.h> + +#include "pg_config.h" +#include "port/pg_bswap.h" + +/* log warnings on backend */ +#ifndef FRONTEND +#define pg_log_warning(...) elog(WARNING, __VA_ARGS__) +#else +#define pg_log_warning(...) (void)0 +#endif + +#ifndef FRONTEND +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define STRDUP(str) pstrdup(str) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define STRDUP(str) strdup(str) +#define FREE(size) free(size) +#endif + +/* ZpqBuffer size, in bytes */ +#define ZPQ_BUFFER_SIZE 8192000 + +#define ZPQ_COMPRESS_THRESHOLD 60 + +/* startup messages have no type field and therefore have a null first byte */ +#define MESSAGE_TYPE_OFFSET(msg_type) (msg_type == '\0' ? 0 : 1) + +typedef struct ZpqBuffer ZpqBuffer; + + +/* ZpqBuffer used as RX/TX buffer in ZpqStream */ +struct ZpqBuffer +{ + char buf[ZPQ_BUFFER_SIZE]; + size_t size; /* current size of buf */ + size_t pos; /* current position in buf, in range [0, size] */ +}; + +/* + * Write up to "src_size" raw (decompressed) bytes. + * Returns number of written raw bytes or error code. + * Error code is either ZPQ_COMPRESS_ERROR or error code returned by the tx function. + * In the last case number of bytes written is stored in *src_processed. + */ +static int zpq_write(IoStreamLayer * layer, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written); + +/* + * Read up to "dst_size" raw (decompressed) bytes. + * Returns number of decompressed bytes or error code. + * Error code is either ZPQ_DECOMPRESS_ERROR or error code returned by the rx function. + */ +static ssize_t zpq_read(IoStreamLayer * layer, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only); + +/* + * Return true if non-flushed data left in internal rx decompression buffer. + */ +static bool zpq_buffered_rx(ZpqStream * zpq); + +/* + * Return true if non-flushed data left in internal tx compression buffer. + */ +static bool zpq_buffered_tx(ZpqStream * zpq); + +/* + * Clear inflight messages and tracked state + */ +static void zpq_reset_write_state(ZpqStream * zpq); + +/* + * Free stream created by zs_create function. + */ +static void zpq_free(ZpqStream * zpq); + +IoStreamProcessor zpq_processor = { + .read = (io_stream_read_func) zpq_read, + .write = (io_stream_write_func) zpq_write, + .buffered_read_data = (io_stream_predicate) zpq_buffered_rx, + .buffered_write_data = (io_stream_predicate) zpq_buffered_tx, + .reset_write_state = (io_stream_consumer) zpq_reset_write_state, + .destroy = (io_stream_consumer) zpq_free +}; + +static inline void +zpq_buf_init(ZpqBuffer * zb) +{ + zb->size = 0; + zb->pos = 0; +} + +static inline size_t +zpq_buf_left(ZpqBuffer * zb) +{ + Assert(zb->buf); + return ZPQ_BUFFER_SIZE - zb->size; +} + +static inline size_t +zpq_buf_unread(ZpqBuffer * zb) +{ + return zb->size - zb->pos; +} + +static inline char * +zpq_buf_size(ZpqBuffer * zb) +{ + return (char *) (zb->buf) + zb->size; +} + +static inline char * +zpq_buf_pos(ZpqBuffer * zb) +{ + return (char *) (zb->buf) + zb->pos; +} + +static inline void +zpq_buf_size_advance(ZpqBuffer * zb, size_t value) +{ + zb->size += value; +} + +static inline void +zpq_buf_pos_advance(ZpqBuffer * zb, size_t value) +{ + zb->pos += value; +} + +static inline void +zpq_buf_reuse(ZpqBuffer * zb) +{ + size_t unread = zpq_buf_unread(zb); + + if (unread > 5) /* can read message header, don't do anything */ + return; + if (unread == 0) + { + zb->size = 0; + zb->pos = 0; + return; + } + memmove(zb->buf, zb->buf + zb->pos, unread); + zb->size = unread; + zb->pos = 0; +} + +struct ZpqStream +{ + ZStream *c_stream; /* underlying compression stream */ + ZStream *d_stream; /* underlying decompression stream */ + + bool is_compressing; /* current compression state */ + + bool is_decompressing; /* current decompression state */ + bool reading_compressed_header; /* compression header processing + * incomplete */ + size_t rx_msg_bytes_left; /* number of bytes left to process without + * changing the decompression state */ + size_t tx_msg_bytes_left; /* number of bytes left to process without + * changing the compression state */ + + ZpqBuffer rx_in; /* buffer for unprocessed data read from next + * stream layer */ + ZpqBuffer tx_in; /* buffer for unprocessed data consumed by + * zpq_write */ + ZpqBuffer tx_out; /* buffer for processed data waiting for send + * to next stream layer */ + + pg_compress_specification *compressors; /* compressors array holds the + * available compressors to use + * for compression/decompression */ + size_t n_compressors; /* size of the compressors array */ + pg_compress_algorithm compress_algs[COMPRESSION_ALGORITHM_COUNT]; /* array of compression + * algorithms supported + * by the reciever of + * the stream */ + pg_compress_algorithm compress_alg; /* active compression algorithm */ + pg_compress_algorithm decompress_alg; /* active decompression algorithm */ +}; + +/* + * Choose the algorithm to use for the message of msg_type with msg_len. + * Returns a pg_compress_algorithm with a registered compressor, or PG_COMPRESSION_NONE if no compressor is appropriate + */ +static inline pg_compress_algorithm +zpq_choose_algorithm(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + /* + * in theory we could choose the algorithm based on the message type + * and/or more complex heuristics, but at least for now we will just use + * the first available algorithm (which defaults to none if compression + * has not yet been enabled) for message types that would most obviously + * benefit from compression + */ + if (msg_len >= ZPQ_COMPRESS_THRESHOLD && (msg_type == PqMsg_CopyData || msg_type == PqMsg_DataRow || msg_type == PqMsg_Query)) + { + return zpq->compress_algs[0]; + } + return PG_COMPRESSION_NONE; +} + +static inline bool +zpq_should_compress(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + return zpq_choose_algorithm(zpq, msg_type, msg_len) != PG_COMPRESSION_NONE; +} + +static inline pg_compress_specification * +zpq_find_compressor(ZpqStream * zpq, pg_compress_algorithm algorithm) +{ + int i; + + for (i = 0; i < zpq->n_compressors; i++) + { + if (algorithm == zpq->compressors[i].algorithm) + return &zpq->compressors[i]; + } + return NULL; +} + +static inline bool +zpq_is_compressed_msg(char msg_type) +{ + return msg_type == PqMsg_CompressedMessage; +} + +ZpqStream * +zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream) +{ + ZpqStream *zpq; + + /* zpqStream needs at least one compressor */ + if (n_compressors == 0 || compressors == NULL) + { + return NULL; + } + zpq = (ZpqStream *) ALLOC(sizeof(ZpqStream)); + /* almost all fields should default to 0/NULL/PG_COMPRESSION_NONE */ + memset(zpq, 0, sizeof(ZpqStream)); + zpq->compressors = compressors; + zpq->n_compressors = n_compressors; + zpq_buf_init(&zpq->tx_in); + zpq_buf_init(&zpq->rx_in); + zpq_buf_init(&zpq->tx_out); + + io_stream_add_layer(stream, &zpq_processor, zpq); + + return zpq; +} + +void +zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms) +{ + Assert(n_algorithms <= COMPRESSION_ALGORITHM_COUNT); + + for (int i = 0; i < COMPRESSION_ALGORITHM_COUNT; i++) + { + if (i < n_algorithms) + { + zpq->compress_algs[i] = algorithms[i]; + } + else + { + zpq->compress_algs[i] = PG_COMPRESSION_NONE; + } + } +} + +/* Compress up to src_size bytes from *src into CompressedData and write it to the tx buffer. + * Returns ZS_OK on success, ZS_COMPRESS_ERROR if encountered a compression error. */ +static inline int +zpq_write_compressed_message(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed) +{ + size_t compressed_len; + ssize_t rc; + uint32 size; + uint8 algorithm; + + /* check if have enough space */ + if (zpq_buf_left(&zpq->tx_out) <= 6) + { + /* too little space for CompressedData, abort */ + *src_processed = 0; + return ZS_OK; + } + + compressed_len = 0; + rc = zs_write(zpq->c_stream, src, src_size, src_processed, + zpq_buf_size(&zpq->tx_out) + 6, zpq_buf_left(&zpq->tx_out) - 6, &compressed_len); + + if (compressed_len > 0) + { + /* write CompressedData type */ + *zpq_buf_size(&zpq->tx_out) = PqMsg_CompressedMessage; + size = pg_hton32(compressed_len + 5); + + memcpy(zpq_buf_size(&zpq->tx_out) + 1, &size, sizeof(uint32)); /* write msg length */ + compressed_len += 6; /* append header length to compressed data + * length */ + algorithm = zpq->compress_alg; + memcpy(zpq_buf_size(&zpq->tx_out) + 5, &algorithm, sizeof(uint8)); /* write msg algorithm */ + } + + zpq_buf_size_advance(&zpq->tx_out, compressed_len); + return rc; +} + +/* Copy the data directly from *src to the tx buffer */ +static void +zpq_write_uncompressed(ZpqStream * zpq, char const *src, size_t src_size, size_t *src_processed) +{ + src_size = Min(zpq_buf_left(&zpq->tx_out), src_size); + memcpy(zpq_buf_size(&zpq->tx_out), src, src_size); + + zpq_buf_size_advance(&zpq->tx_out, src_size); + *src_processed = src_size; +} + +/* Determine if should compress the next message and change the current compression state */ +static int +zpq_toggle_compression(ZpqStream * zpq, char msg_type, uint32 msg_len) +{ + pg_compress_algorithm new_compress_alg = zpq_choose_algorithm(zpq, msg_type, msg_len); + pg_compress_specification *spec; + bool should_compress = new_compress_alg != PG_COMPRESSION_NONE; + + /* + * negative new_compress_idx indicates that we should not compress this + * message + */ + if (should_compress) + { + /* + * if the new compressor does not match the current one, change out + * the underlying z_stream + */ + if (zpq->compress_alg != new_compress_alg) + { + zs_compressor_free(zpq->c_stream); + spec = zpq_find_compressor(zpq, new_compress_alg); + if (spec == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->c_stream = zs_create_compressor(spec); + if (zpq->c_stream == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->compress_alg = new_compress_alg; + } + } + + zpq->is_compressing = should_compress; + zpq->tx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type); + return 0; +} + +/* + * Internal write function. Reads the data from *src buffer, + * determines the postgres messages type and length. + * If message matches the compression criteria, it wraps the message into + * CompressedData. Otherwise, leaves the message unchanged. + * If *src data ends with incomplete message header, this function is not + * going to read this message header. + * Returns 0 on success or error code + * Number of bytes written is stored in *processed. + */ +static int +zpq_write_internal(ZpqStream * zpq, void const *src, size_t src_size, size_t *processed) +{ + size_t src_pos = 0; + ssize_t rc; + + do + { + /* + * try to read ahead the next message types and increase + * tx_msg_bytes_left, if possible + */ + while (zpq->tx_msg_bytes_left > 0 && src_size - src_pos >= zpq->tx_msg_bytes_left + 5) + { + char msg_type = *((char *) src + src_pos + zpq->tx_msg_bytes_left); + uint32 msg_len; + + memcpy(&msg_len, (char *) src + src_pos + zpq->tx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + if (zpq_should_compress(zpq, msg_type, msg_len) != zpq->is_compressing) + { + /* + * cannot proceed further, encountered compression toggle + * point + */ + break; + } + zpq->tx_msg_bytes_left += msg_len + MESSAGE_TYPE_OFFSET(msg_type); + } + + /* + * Write CompressedData if currently is compressing or have some + * buffered data left in underlying compression stream + */ + if (zs_compress_buffered(zpq->c_stream) || (zpq->is_compressing && zpq->tx_msg_bytes_left > 0)) + { + size_t buf_processed = 0; + size_t to_compress = Min(zpq->tx_msg_bytes_left, src_size - src_pos); + + rc = zpq_write_compressed_message(zpq, (char *) src + src_pos, to_compress, &buf_processed); + src_pos += buf_processed; + zpq->tx_msg_bytes_left -= buf_processed; + + if (rc != ZS_OK) + { + *processed = src_pos; + return rc; + } + } + + /* + * If not going to compress the data from *src, just write it + * uncompressed. + */ + else if (zpq->tx_msg_bytes_left > 0) + { /* determine next message type */ + size_t copy_len = Min(src_size - src_pos, zpq->tx_msg_bytes_left); + size_t copy_processed = 0; + + zpq_write_uncompressed(zpq, (char *) src + src_pos, copy_len, ©_processed); + src_pos += copy_processed; + zpq->tx_msg_bytes_left -= copy_processed; + } + + /* + * Reached the compression toggle point, fetch next message header to + * determine compression state. + */ + else + { + char msg_type; + uint32 msg_len; + + if (src_size - src_pos < 5) + { + /* + * must return here because we can't continue without full + * message header + */ + *processed = src_pos; + return 0; + } + + msg_type = *((char *) src + src_pos); + memcpy(&msg_len, (char *) src + src_pos + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + rc = zpq_toggle_compression(zpq, msg_type, msg_len); + if (rc) + { + *processed = src_pos; + return rc; + } + } + + /* + * repeat sending while there is some data in input or internal + * compression buffer + */ + } while (src_pos < src_size && zpq_buf_left(&zpq->tx_out) > 6); + + *processed = src_pos; + return 0; +} + +int +zpq_write(IoStreamLayer * self, ZpqStream * zpq, void const *src, size_t src_size, size_t *bytes_written) +{ + size_t src_pos = 0; + ssize_t rc; + + /* try to process as much data as possible before calling the tx_func */ + while (zpq_buf_left(&zpq->tx_out) > 6) + { + size_t copy_len = Min(zpq_buf_left(&zpq->tx_in), src_size - src_pos); + size_t processed; + + memcpy(zpq_buf_size(&zpq->tx_in), (char *) src + src_pos, copy_len); + zpq_buf_size_advance(&zpq->tx_in, copy_len); + src_pos += copy_len; + + if (zpq_buf_unread(&zpq->tx_in) == 0 && !zs_compress_buffered(zpq->c_stream)) + { + break; + } + + processed = 0; + + rc = zpq_write_internal(zpq, zpq_buf_pos(&zpq->tx_in), zpq_buf_unread(&zpq->tx_in), &processed); + zpq_buf_pos_advance(&zpq->tx_in, processed); + zpq_buf_reuse(&zpq->tx_in); + if (rc < 0) + { + *bytes_written = src_pos; + return rc; + } + if (processed == 0) + { + break; + } + } + *bytes_written = src_pos; + + /* + * call the tx_func if have any bytes to send + */ + while (zpq_buf_unread(&zpq->tx_out)) + { + size_t count; + + rc = io_stream_next_write(self, zpq_buf_pos(&zpq->tx_out), zpq_buf_unread(&zpq->tx_out), &count); + if (!rc && count > 0) + { + zpq_buf_pos_advance(&zpq->tx_out, count); + } + else + { + zpq_buf_reuse(&zpq->tx_out); + return rc; + } + } + + zpq_buf_reuse(&zpq->tx_out); + return 0; +} + +/* Decompress bytes from RX buffer and write up to dst_len of uncompressed data to *dst. + * Returns: + * ZS_OK on success, + * ZS_STREAM_END if reached end of compressed chunk + * ZS_DECOMPRESS_ERROR if encountered a decompression error */ +static inline ssize_t +zpq_read_compressed_message(ZpqStream * zpq, char *dst, size_t dst_len, size_t *dst_processed) +{ + size_t rx_processed = 0; + ssize_t rc; + size_t read_len = Min(zpq->rx_msg_bytes_left, zpq_buf_unread(&zpq->rx_in)); + + Assert(read_len == zpq->rx_msg_bytes_left); + rc = zs_read(zpq->d_stream, zpq_buf_pos(&zpq->rx_in), read_len, &rx_processed, + dst, dst_len, dst_processed); + + zpq_buf_pos_advance(&zpq->rx_in, rx_processed); + zpq->rx_msg_bytes_left -= rx_processed; + return rc; +} + +/* Copy up to dst_len bytes from rx buffer to *dst. + * Returns amount of bytes copied. */ +static inline size_t +zpq_read_uncompressed(ZpqStream * zpq, char *dst, size_t dst_len) +{ + size_t copy_len; + + Assert(zpq_buf_unread(&zpq->rx_in) > 0); + copy_len = Min(zpq->rx_msg_bytes_left, Min(zpq_buf_unread(&zpq->rx_in), dst_len)); + + memcpy(dst, zpq_buf_pos(&zpq->rx_in), copy_len); + + zpq_buf_pos_advance(&zpq->rx_in, copy_len); + zpq->rx_msg_bytes_left -= copy_len; + return copy_len; +} + +/* Determine if should decompress the next message and + * change the current decompression state */ +static inline void +zpq_toggle_decompression(ZpqStream * zpq) +{ + uint32 msg_len; + char msg_type = *zpq_buf_pos(&zpq->rx_in); + + memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + MESSAGE_TYPE_OFFSET(msg_type), 4); + msg_len = pg_ntoh32(msg_len); + + zpq->is_decompressing = zpq_is_compressed_msg(msg_type); + zpq->rx_msg_bytes_left = msg_len + MESSAGE_TYPE_OFFSET(msg_type); + + if (zpq->is_decompressing) + { + /* compressed message header is no longer needed, just skip it */ + zpq_buf_pos_advance(&zpq->rx_in, 5); + zpq->rx_msg_bytes_left -= 5; + zpq->reading_compressed_header = true; + } +} + +static inline ssize_t +zpq_process_switch(ZpqStream * zpq) +{ + pg_compress_algorithm algorithm; + pg_compress_specification *spec; + + if (zpq_buf_unread(&zpq->rx_in) < 1) + { + return 0; + } + + algorithm = *zpq_buf_pos(&zpq->rx_in); + + zpq_buf_pos_advance(&zpq->rx_in, 1); + zpq->reading_compressed_header = false; + zpq->rx_msg_bytes_left -= 1; + + /* + * if the new decompressor does not match the current one, change out the + * underlying z_stream + */ + if (algorithm != zpq->decompress_alg) + { + zs_decompressor_free(zpq->d_stream); + spec = zpq_find_compressor(zpq, algorithm); + if (spec == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->d_stream = zs_create_decompressor(spec); + if (zpq->d_stream == NULL) + { + return ZPQ_FATAL_ERROR; + } + zpq->decompress_alg = algorithm; + } + + return 0; +} + +ssize_t +zpq_read(IoStreamLayer * self, ZpqStream * zpq, void *dst, size_t dst_size, bool buffered_only) +{ + size_t dst_pos = 0; + size_t dst_processed = 0; + ssize_t rc; + + /* Read until some data fetched */ + while (dst_pos == 0) + { + zpq_buf_reuse(&zpq->rx_in); + + if (!zpq_buffered_rx(zpq) || (zpq->is_decompressing && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left)) + { + rc = io_stream_next_read(self, zpq_buf_size(&zpq->rx_in), zpq_buf_left(&zpq->rx_in), buffered_only); + if (rc > 0) /* read fetches some data */ + { + zpq_buf_size_advance(&zpq->rx_in, rc); + } + else if (rc == 0) + { + /* got no more data; return what we have */ + return dst_pos; + } + else /* read failed */ + { + return rc; + } + } + + /* + * try to read ahead the next message types and increase + * rx_msg_bytes_left, if possible (ONLY UNCOMPRESSED MESSAGES) + */ + while (!zpq->is_decompressing && zpq->rx_msg_bytes_left > 0 && (zpq_buf_unread(&zpq->rx_in) >= zpq->rx_msg_bytes_left + 5)) + { + char msg_type; + uint32 msg_len; + + msg_type = *(zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left); + if (zpq_is_compressed_msg(msg_type)) + { + /* + * cannot proceed further, encountered compression toggle + * point + */ + break; + } + + memcpy(&msg_len, zpq_buf_pos(&zpq->rx_in) + zpq->rx_msg_bytes_left + MESSAGE_TYPE_OFFSET(msg_type), 4); + zpq->rx_msg_bytes_left += pg_ntoh32(msg_len) + MESSAGE_TYPE_OFFSET(msg_type); + } + + + /* + * If we are in the middle of reading a message, keep reading it until + * we reach the end at which point we need to check if we should + * toggle compression + */ + if (zpq->rx_msg_bytes_left > 0 || zs_decompress_buffered(zpq->d_stream)) + { + dst_processed = 0; + if (zpq->is_decompressing || zs_decompress_buffered(zpq->d_stream)) + { + if (zpq->reading_compressed_header) + { + zpq_process_switch(zpq); + } + if (!zs_decompress_buffered(zpq->d_stream) && zpq_buf_unread(&zpq->rx_in) < zpq->rx_msg_bytes_left) + { + /* + * prefer to read only the fully compressed messages or + * read if some data is buffered + */ + continue; + } + rc = zpq_read_compressed_message(zpq, dst, dst_size - dst_pos, &dst_processed); + dst_pos += dst_processed; + if (rc == ZS_STREAM_END) + { + continue; + } + if (rc != ZS_OK) + { + return rc; + } + } + else + dst_pos += zpq_read_uncompressed(zpq, dst, dst_size - dst_pos); + } + else if (zpq_buf_unread(&zpq->rx_in) >= 5) + zpq_toggle_decompression(zpq); + } + return dst_pos; +} + +bool +zpq_buffered_rx(ZpqStream * zpq) +{ + return zpq ? zpq_buf_unread(&zpq->rx_in) >= 5 || (zpq_buf_unread(&zpq->rx_in) > 0 && zpq->rx_msg_bytes_left > 0) || + zs_decompress_buffered(zpq->d_stream) : 0; +} + +bool +zpq_buffered_tx(ZpqStream * zpq) +{ + return zpq ? zpq_buf_unread(&zpq->tx_in) >= 5 || (zpq_buf_unread(&zpq->tx_in) > 0 && zpq->tx_msg_bytes_left > 0) || zpq_buf_unread(&zpq->tx_out) > 0 || + zs_compress_buffered(zpq->c_stream) : 0; +} + +void +zpq_reset_write_state(ZpqStream * zpq) +{ + /* + * We need to flush where we are in the msg to ensure we keep our reads + * correctly aligned + */ + zpq->tx_msg_bytes_left = 0; + zpq->tx_in.pos = 0; + zpq->tx_in.size = 0; + zpq->tx_out.pos = 0; + zpq->tx_out.size = 0; + + /* + * Disable compression for the rest of the life of this connection since + * any existing state will be stale, and we only call this after a failure + * at which point the connection will be terminated shortly anyways + */ + zpq->is_compressing = false; + for (int i = 0; i < COMPRESSION_ALGORITHM_COUNT; i++) + { + zpq->compress_algs[i] = PG_COMPRESSION_NONE; + } +} + +void +zpq_free(ZpqStream * zpq) +{ + if (zpq) + { + if (zpq->c_stream) + { + zs_compressor_free(zpq->c_stream); + } + if (zpq->d_stream) + { + zs_decompressor_free(zpq->d_stream); + } + FREE(zpq); + } +} + +char const * +zpq_compress_error(ZpqStream * zpq) +{ + return zs_compress_error(zpq->c_stream); +} + +char const * +zpq_decompress_error(ZpqStream * zpq) +{ + return zs_decompress_error(zpq->d_stream); +} + +pg_compress_algorithm +zpq_compress_algorithm(ZpqStream * zpq) +{ + return zs_algorithm(zpq->c_stream); +} + +pg_compress_algorithm +zpq_decompress_algorithm(ZpqStream * zpq) +{ + return zs_algorithm(zpq->d_stream); +} + +char * +zpq_algorithms(ZpqStream * zpq) +{ + return zpq_serialize_compressors(zpq->compressors, zpq->n_compressors); +} + +int +zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors) +{ + int i; + + *n_compressors = 0; + memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT); + + if (pg_strcasecmp(val, "true") == 0 || + pg_strcasecmp(val, "yes") == 0 || + pg_strcasecmp(val, "on") == 0 || + pg_strcasecmp(val, "1") == 0) + { + int j = 0; + + /* + * return all available compressors, processing NONE (0) as the last + * algorithm rather than the first + */ + for (i = 1; i <= COMPRESSION_ALGORITHM_COUNT; i++) + { + if (supported_compression_algorithm(i % COMPRESSION_ALGORITHM_COUNT)) + *n_compressors += 1; + } + + for (i = 1; i <= COMPRESSION_ALGORITHM_COUNT; i++) + { + if (supported_compression_algorithm(i % COMPRESSION_ALGORITHM_COUNT)) + { + parse_compress_specification(i % COMPRESSION_ALGORITHM_COUNT, NULL, &compressors[j]); + j += 1; + } + } + return 1; + } + + if (*val == 0 || + pg_strcasecmp(val, "false") == 0 || + pg_strcasecmp(val, "no") == 0 || + pg_strcasecmp(val, "off") == 0 || + pg_strcasecmp(val, "0") == 0) + { + /* Compression is disabled */ + return 0; + } + + return zpq_deserialize_compressors(val, compressors, n_compressors) ? 1 : -1; +} + +bool +zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors) +{ + int selected_alg_mask = 0; /* bitmask of already selected + * algorithms to avoid duplicates in + * compressors */ + char *c_string_dup = STRDUP(c_string); /* following parsing can + * modify the string */ + char *p = c_string_dup; + + *n_compressors = 0; + memset(compressors, 0, sizeof(pg_compress_specification) * COMPRESSION_ALGORITHM_COUNT); + + while (*p != '\0') + { + char *sep = strchr(p, ';'); + char *col; + char *error_detail; + pg_compress_algorithm algorithm; + pg_compress_specification *spec = &compressors[*n_compressors]; + + if (sep != NULL) + *sep = '\0'; + + col = strchr(p, ':'); + if (col != NULL) + { + *col = '\0'; + } + if (!parse_compress_algorithm(p, &algorithm)) + { + pg_log_warning("invalid compression algorithm %s", p); + goto error; + } + + if (supported_compression_algorithm(algorithm)) + { + parse_compress_specification(algorithm, col == NULL ? NULL : col + 1, spec); + error_detail = validate_compress_specification(spec); + if (error_detail) + { + pg_log_warning("invalid compression specification: %s", error_detail); + goto error; + } + if (spec->options & PG_COMPRESSION_OPTION_WORKERS || spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) + { + pg_log_warning("streaming compression does not support workers or long distance options"); + goto error; + } + + if (selected_alg_mask & (1 << algorithm)) + { + /* duplicates are not allowed */ + pg_log_warning("duplicate algorithm %s in compressors string %s", get_compress_algorithm_name(algorithm), c_string); + goto error; + } + + *n_compressors += 1; + selected_alg_mask |= 1 << algorithm; + } + else + { + /* + * Intentionally do not return an error, as we must support + * clients/servers parsing algorithms they don't suppport mixed + * with ones they do support + */ + pg_log_warning("this build does not support compression with %s", + get_compress_algorithm_name(algorithm)); + } + + if (sep) + p = sep + 1; + else + break; + } + + /* + * Always add NONE to the list at the end to allow one-directional + * compression, but only if there are other algorithms available. If there + * are no active compressors available, we should return n_compressors = 0 + * to allow other layers to skip adding the compression processing layers + */ + if (n_compressors > 0 && !(selected_alg_mask & (1 << PG_COMPRESSION_NONE))) + { + pg_compress_specification *spec = &compressors[*n_compressors]; + + spec->algorithm = PG_COMPRESSION_NONE; + *n_compressors += 1; + } + + FREE(c_string_dup); + return true; + +error: + FREE(c_string_dup); + *n_compressors = 0; + return false; +} + +/* + * Because deserialize rejects all options, this method only needs to concern + * itself with only serializing name and level + */ +char * +zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors) +{ + char *res; + char *p; + size_t i; + size_t total_len = 0; + + if (n_compressors == 0) + { + return NULL; + } + + for (i = 0; i < n_compressors; i++) + { + size_t level_len; + + /* determine the length of the compression level string */ + level_len = compressors[i].level == 0 ? 1 : (int) floor(log10(abs(compressors[i].level))) + 1; + if (compressors[i].level < 0) + { + level_len += 1; /* add the leading "-" */ + } + + /* + * single entry looks like "alg_name:compression_level," so +2 is for + * ":" and ";" symbols (or trailing null) + */ + total_len += strlen(get_compress_algorithm_name(compressors[i].algorithm)) + level_len + 2; + } + + res = p = ALLOC(total_len); + + for (i = 0; i < n_compressors; i++) + { + p += sprintf(p, "%s:%d", get_compress_algorithm_name(compressors[i].algorithm), compressors[i].level); + if (i < n_compressors - 1) + *p++ = ';'; + } + return res; +} diff --git a/src/include/common/compression.h b/src/include/common/compression.h index c94ace6e8a..9c7d68a036 100644 --- a/src/include/common/compression.h +++ b/src/include/common/compression.h @@ -17,6 +17,7 @@ /* * These values are stored in disk, for example in files generated by pg_dump. * Create the necessary backwards compatibility layers if their order changes. + * Make sure to keep COMPRESSION_ALGORITHM_COUNT in sync if adding new values. */ typedef enum pg_compress_algorithm { @@ -26,6 +27,8 @@ typedef enum pg_compress_algorithm PG_COMPRESSION_ZSTD, } pg_compress_algorithm; +#define COMPRESSION_ALGORITHM_COUNT (PG_COMPRESSION_ZSTD + 1) + #define PG_COMPRESSION_OPTION_WORKERS (1 << 0) #define PG_COMPRESSION_OPTION_LONG_DISTANCE (1 << 1) @@ -36,7 +39,9 @@ typedef struct pg_compress_specification int level; int workers; bool long_distance; - char *parse_error; /* NULL if parsing was OK, else message */ + bool has_error; + char parse_error[255]; /* Populated with error message if + * has_error is true */ } pg_compress_specification; extern void parse_compress_options(const char *option, char **algorithm, @@ -49,5 +54,6 @@ extern void parse_compress_specification(pg_compress_algorithm algorithm, pg_compress_specification *result); extern char *validate_compress_specification(pg_compress_specification *); +extern bool supported_compression_algorithm(pg_compress_algorithm algorithm); #endif diff --git a/src/include/common/io_stream.h b/src/include/common/io_stream.h index 9af88aa9ea..77b8af71d6 100644 --- a/src/include/common/io_stream.h +++ b/src/include/common/io_stream.h @@ -23,7 +23,7 @@ typedef struct IoStream IoStream; typedef ssize_t (*io_stream_read_func) (IoStreamLayer * self, void *context, void *data, size_t size, bool buffered_only); typedef int (*io_stream_write_func) (IoStreamLayer * self, void *context, void const *data, size_t size, size_t *bytes_written); typedef bool (*io_stream_predicate) (void *context); -typedef void (*io_stream_destroy_func) (void *context); +typedef void (*io_stream_consumer) (void *context); typedef struct IoStreamProcessor { @@ -49,11 +49,18 @@ typedef struct IoStreamProcessor */ io_stream_predicate buffered_write_data; + /* + * Optional Will be called if data being sent to the stream has been reset + * early (e.g. due to a transmission error). Only necessary if the + * processor inspects the messages of the stream and tracks related state + */ + io_stream_consumer reset_write_state; + /* * Optional will be called as part of io_stream_destroy when cleaning up * the stream */ - io_stream_destroy_func destroy; + io_stream_consumer destroy; } IoStreamProcessor; /* @@ -108,6 +115,12 @@ extern bool io_stream_buffered_read_data(IoStream * stream); */ extern bool io_stream_buffered_write_data(IoStream * stream); +/* + * Resets any state in the processors about currently in-flight messages + * Should be called if message transmission is aborted for any reason + */ +extern void io_stream_reset_write_state(IoStream * stream); + /* * Read data from the next layer of the stream * (to be used by io_stream_read_func) diff --git a/src/include/common/z_stream.h b/src/include/common/z_stream.h new file mode 100644 index 0000000000..e456d509e1 --- /dev/null +++ b/src/include/common/z_stream.h @@ -0,0 +1,98 @@ +/*------------------------------------------------------------------------- + * + * z_stream.h + * Streaming compression algorithms + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/z_stream.h + * + *------------------------------------------------------------------------- + */ +#ifndef Z_STREAM_H +#define Z_STREAM_H + +#include <stdlib.h> + +#include "compression.h" + +#define ZS_OK (0) +#define ZS_IO_ERROR (-1) +#define ZS_DECOMPRESS_ERROR (-2) +#define ZS_COMPRESS_ERROR (-3) +#define ZS_STREAM_END (-4) +#define ZS_INCOMPLETE_SRC (-5) /* cannot decompress unless full src message + * is fetched */ + +struct ZStream; +typedef struct ZStream ZStream; + +#endif + +/* + * Create compression stream for sending compressed data. + * spec: specifications of chosen decompression algorithm + */ +extern ZStream * zs_create_compressor(pg_compress_specification *spec); + +/* + * Create decompression stream for reading compressed data. + * spec: specifications of chosen decompression algorithm + */ +extern ZStream * zs_create_decompressor(pg_compress_specification *spec); + +/* + * Read up to "size" raw (decompressed) bytes. + * Returns ZS_OK on success or error code. + * Stores bytes read from src in src_processed, bytes written to dst in dst_process. + */ +extern int zs_read(ZStream * zs, void const *src, size_t src_size, size_t *src_processed, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Write up to "size" raw (decompressed) bytes. + * Returns number of written raw bytes or error code. + * Returns ZS_OK on success or error code. + * Stores bytes read from buf in processed, bytes written to dst in dst_process + */ +extern int zs_write(ZStream * zs, void const *buf, size_t size, size_t *processed, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Returns if there is buffered raw data remaining in the stream to compress + */ +extern bool zs_compress_buffered(ZStream * zs); + +/* + * Returns if there is buffered decompressed data remaining in the stream to read + */ +extern bool zs_decompress_buffered(ZStream * zs); + +/* + * Get decompressor error message. + */ +extern char const *zs_decompress_error(ZStream * zs); + +/* + * Get compressor error message. + */ +extern char const *zs_compress_error(ZStream * zs); + +/* + * End the compression stream. + */ +extern int zs_end_compression(ZStream * zs, void *dst, size_t dst_size, size_t *dst_processed); + +/* + * Free stream created by zs_create_compressor function. + */ +extern void zs_compressor_free(ZStream * zs); + +/* + * Free stream created by zs_create_decompressor function. + */ +extern void zs_decompressor_free(ZStream * zs); + +/* + * Get the descriptor of chosen algorithm. + */ +extern pg_compress_algorithm zs_algorithm(ZStream * zs); diff --git a/src/include/common/zpq_stream.h b/src/include/common/zpq_stream.h new file mode 100644 index 0000000000..15570f2363 --- /dev/null +++ b/src/include/common/zpq_stream.h @@ -0,0 +1,91 @@ +/*------------------------------------------------------------------------- + * + * zpq_stream.h + * IO stream layer applying ZStream compression to libpq + * + * Copyright (c) 2018-2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/zpq_stream.h + * + *------------------------------------------------------------------------- + */ +#include "io_stream.h" +#include "z_stream.h" + +#ifndef ZPQ_STREAM_H +#define ZPQ_STREAM_H + +#define ZPQ_FATAL_ERROR (-7) +struct ZpqStream; +typedef struct ZpqStream ZpqStream; + +#endif + +/* + * Create compression stream with rx/tx function for reading/sending compressed data. + * io_stream: IO Stream to layer on top of + * rx_data: received data (compressed data already fetched from input stream) + * rx_data_size: size of data fetched from input stream + * The returned ZpqStream can only be destroyed by destoing the IoStream with io_stream_destroy. + */ +extern ZpqStream * zpq_create(pg_compress_specification *compressors, size_t n_compressors, IoStream * stream); + +/* + * Start compressing applicable outgoing data once the connection is sufficiently set up + */ +extern void zpq_enable_compression(ZpqStream * zpq, pg_compress_algorithm *algorithms, size_t n_algorithms); + +/* + * Get decompressor error message. + */ +extern char const *zpq_decompress_error(ZpqStream * zpq); + +/* + * Get compressor error message. + */ +extern char const *zpq_compress_error(ZpqStream * zpq); + +/* + * Get the name of the current compression algorithm. + */ +extern pg_compress_algorithm zpq_compress_algorithm(ZpqStream * zpq); + +/* + * Get the name of the current decompression algorithm. + */ +extern pg_compress_algorithm zpq_decompress_algorithm(ZpqStream * zpq); + +/* + * Parse the compression setting. + * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT + * Returns: + * - 1 if the compression setting is valid + * - 0 if the compression setting is valid but disabled + * - -1 if the compression setting is invalid + * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors. + * If no supported compressors recognized or if compression is disabled, then n_compressors is set to 0. + */ +extern int + zpq_parse_compression_setting(const char *val, pg_compress_specification *compressors, size_t *n_compressors); + +/* Serialize the compressors array to string so it can be transmitted to the other side during the compression startup. + * For example, for array of two compressors (zstd, level 1), (zlib, level 2) resulting string would look like "zstd:1;zlib:2". + * Returns the resulting string. + */ +extern char + *zpq_serialize_compressors(pg_compress_specification const *compressors, size_t n_compressors); + +/* Deserialize the compressors string received during the compression setup to a compressors array. + * Compressors must be an array of length COMPRESSION_ALGORITHM_COUNT + * Returns: + * - true if the compressors string is successfully parsed + * - false otherwise + * It also populates the compressors array with the recognized compressors. Size of the array is stored in n_compressors. + * If no supported compressors are recognized or c_string is empty, then n_compressors is set to 0. + */ +bool + zpq_deserialize_compressors(char const *c_string, pg_compress_specification *compressors, size_t *n_compressors); + +/* Returns the currently enabled compression algorithms using zpq_serialize_compressors */ +char *zpq_algorithms(ZpqStream * zpq); diff --git a/src/include/libpq/compression.h b/src/include/libpq/compression.h new file mode 100644 index 0000000000..927ef42751 --- /dev/null +++ b/src/include/libpq/compression.h @@ -0,0 +1,30 @@ +/*------------------------------------------------------------------------- + * + * compression.h + * Interface to libpq/compression.c + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * src/include/libpq/compression.h + * + *------------------------------------------------------------------------- + */ +#ifndef LIBPQ_COMPRESSION_H +#define LIBPQ_COMPRESSION_H + +#include "postgres.h" +#include "libpq-be.h" +#include "common/compression.h" + +extern PGDLLIMPORT char *libpq_compress_algorithms; +extern PGDLLIMPORT pg_compress_specification libpq_compressors[COMPRESSION_ALGORITHM_COUNT]; +extern PGDLLIMPORT size_t libpq_n_compressors; + +/* + * Enbles compression processing on the given port. + * val is the value of the _pq_.libpq_compression startup packet parameter + */ +extern void configure_libpq_compression(Port *port, const char *val); + +#endif /* LIBPQ_COMPRESSION_H */ diff --git a/src/include/libpq/libpq-be-fe-helpers.h b/src/include/libpq/libpq-be-fe-helpers.h index 41e3bb4376..cd840391f0 100644 --- a/src/include/libpq/libpq-be-fe-helpers.h +++ b/src/include/libpq/libpq-be-fe-helpers.h @@ -206,11 +206,14 @@ libpqsrv_connect_internal(PGconn *conn, uint32 wait_event_info) else io_flag = WL_SOCKET_WRITEABLE; - rc = WaitLatchOrSocket(MyLatch, - WL_EXIT_ON_PM_DEATH | WL_LATCH_SET | io_flag, - PQsocket(conn), - 0, - wait_event_info); + if (PQreadPending(conn)) + rc = WL_SOCKET_READABLE; + else + rc = WaitLatchOrSocket(MyLatch, + WL_EXIT_ON_PM_DEATH | WL_LATCH_SET | io_flag, + PQsocket(conn), + 0, + wait_event_info); /* Interrupted? */ if (rc & WL_LATCH_SET) diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index 87ba7f5ea0..ec3b69351c 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -53,6 +53,8 @@ typedef struct #endif #endif /* ENABLE_SSPI */ +#include <common/zpq_stream.h> + #include "common/io_stream.h" #include "datatype/timestamp.h" #include "libpq/hba.h" @@ -161,6 +163,7 @@ typedef struct Port int remote_hostname_errcode; /* see above */ char *remote_port; /* text rep of remote port */ CAC_state canAcceptConnections; /* postmaster connection status */ + ZpqStream *zpq_stream; /* streaming compression state */ /* * Information that needs to be saved from the startup packet and passed diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h index 3612280146..01cc3911fa 100644 --- a/src/include/libpq/libpq.h +++ b/src/include/libpq/libpq.h @@ -73,6 +73,7 @@ extern void StreamClose(pgsocket sock); extern void TouchSocketFiles(void); extern void RemoveSocketFiles(void); extern void pq_init(void); +extern int pq_configure(Port *port); extern int pq_getbytes(char *s, size_t len); extern void pq_startmsgread(void); extern void pq_endmsgread(void); diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h index cc46f4b586..d9eaebbf7e 100644 --- a/src/include/libpq/protocol.h +++ b/src/include/libpq/protocol.h @@ -63,7 +63,7 @@ #define PqMsg_CopyDone 'c' #define PqMsg_CopyData 'd' - +#define PqMsg_CompressedMessage 'z' /* These are the authentication request codes sent by the backend. */ diff --git a/src/include/utils/guc_hooks.h b/src/include/utils/guc_hooks.h index 3d74483f44..ee34fbb20e 100644 --- a/src/include/utils/guc_hooks.h +++ b/src/include/utils/guc_hooks.h @@ -63,6 +63,8 @@ extern bool check_effective_io_concurrency(int *newval, void **extra, GucSource source); extern bool check_huge_page_size(int *newval, void **extra, GucSource source); extern const char *show_in_hot_standby(void); +extern bool check_libpq_compression(char **newval, void **extra, GucSource source); +extern void assign_libpq_compression(const char *newval, void *extra); extern bool check_locale_messages(char **newval, void **extra, GucSource source); extern void assign_locale_messages(const char *newval, void *extra); extern bool check_locale_monetary(char **newval, void **extra, GucSource source); diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index 850734ac96..392a6815e3 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -191,3 +191,6 @@ PQclosePrepared 188 PQclosePortal 189 PQsendClosePrepared 190 PQsendClosePortal 191 +PQcompression 192 +PQreadPending 193 +PQserverCompression 194 diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index a0f12e62af..2356b9719c 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -25,6 +25,7 @@ #include "common/ip.h" #include "common/link-canary.h" #include "common/scram-common.h" +#include "common/zpq_stream.h" #include "common/string.h" #include "fe-auth.h" #include "libpq-fe.h" @@ -398,6 +399,10 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Replication", "D", 5, offsetof(struct pg_conn, replication)}, + {"compression", "PGCOMPRESSION", "off", NULL, + "Libpq-compression", "", 16, + offsetof(struct pg_conn, compression)}, + {"target_session_attrs", "PGTARGETSESSIONATTRS", DefaultTargetSessionAttrs, NULL, "Target-Session-Attrs", "", 15, /* sizeof("prefer-standby") = 15 */ @@ -523,6 +528,7 @@ pqDropConnection(PGconn *conn, bool flushInput) { io_stream_destroy(conn->io_stream); conn->io_stream = NULL; + conn->zpqStream = NULL; /* Close the socket itself */ if (conn->sock != PGINVALID_SOCKET) closesocket(conn->sock); @@ -1713,6 +1719,34 @@ connectOptions2(PGconn *conn) goto oom_error; } + /* + * validate compression option + */ + if (conn->compression && conn->compression[0]) + { + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + int rc = zpq_parse_compression_setting(conn->compression, compressors, &n_compressors); + + if (rc == -1) + { + conn->status = CONNECTION_BAD; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("invalid %s value: \"%s\"\n"), + "compression", conn->compression); + return false; + } + + if (rc == 1 && n_compressors == 0) + { + conn->status = CONNECTION_BAD; + appendPQExpBuffer(&conn->errorMessage, + libpq_gettext("no supported algorithms found, %s value: \"%s\"\n"), + "compression", conn->compression); + return false; + } + } + /* * validate target_session_attrs option, and set target_server_type */ @@ -3347,6 +3381,20 @@ keep_going: /* We will come back to here until there is goto error_return; } + /* + * By this point we have set up any encryption layers needed, + * so we can inject compression processing if requested. + */ + if (!conn->zpqStream && conn->n_compressors > 0) + { + conn->zpqStream = zpq_create(conn->compressors, conn->n_compressors, conn->io_stream); + if (!conn->zpqStream) + { + libpq_append_conn_error(conn, "failed to set up compression stream"); + goto error_return; + } + } + /* * Send the startup packet. * @@ -3834,14 +3882,22 @@ keep_going: /* We will come back to here until there is } else if (beresp == PqMsg_NegotiateProtocolVersion) { - if (pqGetNegotiateProtocolVersion3(conn)) + if ((res = pqProcessNegotiateProtocolVersion3(conn)) < 0) { libpq_append_conn_error(conn, "received invalid protocol negotiation message"); goto error_return; } /* OK, we read the message; mark data consumed */ conn->inStart = conn->inCursor; - goto error_return; + + if (res == 0) + { + goto error_return; + } + else + { + goto keep_going; + } } /* It is an authentication request. */ @@ -4270,6 +4326,50 @@ error_return: return PGRES_POLLING_FAILED; } +/* + * Based on server GUC libpq_compression, establishes a zpq_stream to compres + * protocol traffic. Should only be called once per connection lifecycle, as the + * zpq_stream cannot be reconfigured without restarting the connection + */ +void +pqConfigureCompression(PGconn *conn, const char *compressors_str) +{ + pg_compress_specification be_compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_be_compressors; + + zpq_deserialize_compressors(compressors_str, be_compressors, &n_be_compressors); + if (conn->zpqStream) + { + pg_compress_algorithm algorithms[COMPRESSION_ALGORITHM_COUNT]; + size_t n_algorithms = 0; + + if (n_be_compressors == 0) + { + return; + } + + /* + * Intersect client and server compressors to determine the final list + * of the supported compressors. O(N^2) is negligible because of a + * small number of the compression methods. + */ + for (size_t i = 0; i < conn->n_compressors; i++) + { + for (size_t j = 0; j < n_be_compressors; j++) + { + if (conn->compressors[i].algorithm == be_compressors[j].algorithm) + { + algorithms[n_algorithms] = conn->compressors[i].algorithm; + n_algorithms += 1; + break; + } + } + } + + zpq_enable_compression(conn->zpqStream, algorithms, n_algorithms); + } +} + /* * internal_ping @@ -4480,6 +4580,7 @@ freePGconn(PGconn *conn) free(conn->fbappname); free(conn->dbName); free(conn->replication); + free(conn->compression); free(conn->pguser); if (conn->pgpass) { @@ -7163,6 +7264,24 @@ PQuser(const PGconn *conn) return conn->pguser; } +char * +PQcompression(const PGconn *conn) +{ + if (!conn || !conn->zpqStream) + return NULL; + + return zpq_algorithms(conn->zpqStream); +} + +char * +PQserverCompression(const PGconn *conn) +{ + if (!conn) + return NULL; + + return conn->server_compression; +} + char * PQpass(const PGconn *conn) { @@ -8048,7 +8167,8 @@ retry_masked: strlcat(msgbuf, "\n", sizeof(msgbuf)); conn->write_err_msg = strdup(msgbuf); /* Now claim the write succeeded */ - n = len; + *bytes_written = n; + n = 0; break; default: @@ -8063,7 +8183,8 @@ retry_masked: strlcat(msgbuf, "\n", sizeof(msgbuf)); conn->write_err_msg = strdup(msgbuf); /* Now claim the write succeeded */ - n = len; + *bytes_written = n; + n = 0; break; } } diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index b9511df2c2..b6158d29c2 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -1185,6 +1185,15 @@ pqSaveParameterStatus(PGconn *conn, const char *name, const char *value) { conn->scram_sha_256_iterations = atoi(value); } + else if (strcmp(name, "libpq_compression") == 0) + { + if (conn->server_compression) + { + free(conn->server_compression); + } + conn->server_compression = strdup(value); + pqConfigureCompression(conn, value); + } } @@ -3962,6 +3971,12 @@ pqPipelineFlush(PGconn *conn) return 0; } +int +PQreadPending(PGconn *conn) +{ + return pqReadPending(conn); +} + /* * PQfreemem - safely frees memory allocated diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index 6772f2876d..9aeeb123b7 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -51,6 +51,8 @@ #include "pg_config_paths.h" #include "port/pg_bswap.h" +#include <common/zpq_stream.h> + static int pqPutMsgBytes(const void *buf, size_t len, PGconn *conn); static int pqSendSome(PGconn *conn, int len); static int pqSocketCheck(PGconn *conn, int forRead, int forWrite, @@ -618,6 +620,14 @@ retry3: conn->inBufSize - conn->inEnd, false); if (nread < 0) { + if (nread == ZS_DECOMPRESS_ERROR) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("decompress error: %s\n"), + zpq_decompress_error(conn->zpqStream)); + return -1; + } + switch (SOCK_ERRNO) { case EINTR: @@ -713,6 +723,14 @@ retry4: conn->inBufSize - conn->inEnd, false); if (nread < 0) { + if (nread == ZS_DECOMPRESS_ERROR) + { + printfPQExpBuffer(&conn->errorMessage, + libpq_gettext("decompress error: %s\n"), + zpq_decompress_error(conn->zpqStream)); + return -1; + } + switch (SOCK_ERRNO) { case EINTR: @@ -822,7 +840,7 @@ pqSendSome(PGconn *conn, int len) } /* while there's still data to send */ - while (len > 0) + while (len > 0 || io_stream_buffered_write_data(conn->io_stream)) { size_t sent; int rc; @@ -883,7 +901,7 @@ pqSendSome(PGconn *conn, int len) } } - if (len > 0) + if (len > 0 || rc < 0 || io_stream_buffered_write_data(conn->io_stream)) { /* * We didn't send it all, wait till we can send more. @@ -951,7 +969,7 @@ pqSendSome(PGconn *conn, int len) int pqFlush(PGconn *conn) { - if (conn->outCount > 0) + if (conn->outCount > 0 || io_stream_buffered_write_data(conn->io_stream)) { if (conn->Pfdebug) fflush(conn->Pfdebug); @@ -976,6 +994,8 @@ pqFlush(PGconn *conn) int pqWait(int forRead, int forWrite, PGconn *conn) { + if (forRead && conn->inCursor < conn->inEnd) + return 0; return pqWaitTimed(forRead, forWrite, conn, (time_t) -1); } @@ -1030,8 +1050,9 @@ pqWriteReady(PGconn *conn) * or both. Returns >0 if one or more conditions are met, 0 if it timed * out, -1 if an error occurred. * - * If SSL is in use, the SSL buffer is checked prior to checking the socket - * for read data directly. + * If protocol layers are using buffering, the buffers are checked prior + * to checking the socket for read data directly. + * */ static int pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) @@ -1068,6 +1089,22 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) return result; } +/* + * Check if there is some data pending in stream buffers. + * Returns -1 on failure, 0 if no, 1 if yes. + */ +int +pqReadPending(PGconn *conn) +{ + if (!conn) + return -1; + + if (io_stream_buffered_read_data(conn->io_stream)) + /* short-circuit the select */ + return 1; + + return 0; +} /* * Check a file descriptor for read and/or write data, possibly waiting. diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index 8c4ec079ca..9d4642cfb4 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -74,6 +74,20 @@ pqParseInput3(PGconn *conn) */ for (;;) { + /* + * Read any available buffered data + * io_stream may be null when draining the final messages from an aborted connection + */ + if (conn->io_stream && pqReadPending(conn) && (conn->inBufSize - conn->inEnd > 0)) + { + int rc = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, conn->inBufSize - conn->inEnd, true); + + if (rc > 0) + { + conn->inEnd += rc; + } + } + /* * Try to read a message. First get the type code and length. Return * if not enough data. @@ -1404,16 +1418,18 @@ reportErrorPosition(PQExpBuffer msg, const char *query, int loc, int encoding) /* * Attempt to read a NegotiateProtocolVersion message. * Entry: 'v' message type and length have already been consumed. - * Exit: returns 0 if successfully consumed message. + * Exit: returns 0 if successfully consumed message and should exit, + * returns 1 if successfully consumed message and should continue with warning, * returns EOF if not enough data. */ int -pqGetNegotiateProtocolVersion3(PGconn *conn) +pqProcessNegotiateProtocolVersion3(PGconn *conn) { int tmp; ProtocolVersion their_version; int num; PQExpBufferData buf; + int retval = 1; if (pqGetInt(&tmp, 4, conn) != 0) return EOF; @@ -1436,9 +1452,12 @@ pqGetNegotiateProtocolVersion3(PGconn *conn) } if (their_version < conn->pversion) + { libpq_append_conn_error(conn, "protocol version not supported by server: client uses %u.%u, server supports up to %u.%u", PG_PROTOCOL_MAJOR(conn->pversion), PG_PROTOCOL_MINOR(conn->pversion), PG_PROTOCOL_MAJOR(their_version), PG_PROTOCOL_MINOR(their_version)); + retval = 0; + } if (num > 0) { appendPQExpBuffer(&conn->errorMessage, @@ -1446,14 +1465,26 @@ pqGetNegotiateProtocolVersion3(PGconn *conn) "protocol extensions not supported by server: %s", num), buf.data); appendPQExpBufferChar(&conn->errorMessage, '\n'); + + /* + * We can continue to use the connection if the server doesn't support + * compression + */ + if (num > 1 || (strcmp(buf.data, "_pq_.libpq_compression") != 0)) + { + retval = 0; + } } /* neither -- server shouldn't have sent it */ if (!(their_version < conn->pversion) && !(num > 0)) + { libpq_append_conn_error(conn, "invalid %s message", "NegotiateProtocolVersion"); + retval = 0; + } termPQExpBuffer(&buf); - return 0; + return retval; } @@ -1764,7 +1795,7 @@ pqGetCopyData3(PGconn *conn, char **buffer, int async) if (msgLength == 0) { /* Don't block if async read requested */ - if (async) + if (async && !pqReadPending(conn)) return 0; /* Need to load more data */ if (pqWait(true, false, conn) || @@ -2246,6 +2277,46 @@ pqBuildStartupPacket3(PGconn *conn, int *packetlen, return startpacket; } +/* + * Build semicolon-separated list of compression algorithms requested by client. + * It can be either explicitly specified by user in connection string, or + * include all algorithms supported by client library. + * This function returns true if the compression string is successfully parsed and + * stores a comma-separated list of algorithms in *client_compressors. + * If compression is disabled, then NULL is assigned to *client_compressors. + * Also it creates an array of compressor descriptors, each element of which corresponds to + * the corresponding algorithm name in *client_compressors list. This array is stored in PGconn + * and is used during handshake when a compression acknowledgment response is received from the server. + */ +static bool +build_compressors_list(PGconn *conn, char **client_compressors, bool build_descriptors) +{ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; + size_t n_compressors; + + if (zpq_parse_compression_setting(conn->compression, compressors, &n_compressors) < 0) + { + return false; + } + + *client_compressors = NULL; + if (build_descriptors) + { + memcpy(conn->compressors, compressors, sizeof(compressors)); + conn->n_compressors = n_compressors; + } + + if (n_compressors == 0) + { + /* no compressors available, return */ + return true; + } + + *client_compressors = zpq_serialize_compressors(compressors, n_compressors); + + return true; +} + /* * Build a startup packet given a filled-in PGconn structure. * @@ -2292,6 +2363,19 @@ build_startup_packet(const PGconn *conn, char *packet, ADD_STARTUP_OPTION("replication", conn->replication); if (conn->pgoptions && conn->pgoptions[0]) ADD_STARTUP_OPTION("options", conn->pgoptions); + if (conn->compression && conn->compression[0]) + { + char *client_compression_algorithms; + + if (build_compressors_list((PGconn *) conn, &client_compression_algorithms, packet == NULL)) + { + if (client_compression_algorithms) + { + ADD_STARTUP_OPTION("_pq_.libpq_compression", client_compression_algorithms); + free(client_compression_algorithms); + } + } + } if (conn->send_appname) { /* Use appname if present, otherwise use fallback */ diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index b9706cb151..69a2c86c7f 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -90,7 +90,7 @@ IoStreamProcessor pgtls_processor = { .read = (io_stream_read_func) pgtls_read, .write = (io_stream_write_func) pgtls_write, .buffered_read_data = (io_stream_predicate) pgtls_read_pending, - .destroy = (io_stream_destroy_func) pgtls_close + .destroy = (io_stream_consumer) pgtls_close }; static bool pq_init_ssl_lib = true; diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 97762d56f5..fb7d8a9471 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -342,6 +342,8 @@ extern char *PQhostaddr(const PGconn *conn); extern char *PQport(const PGconn *conn); extern char *PQtty(const PGconn *conn); extern char *PQoptions(const PGconn *conn); +extern char *PQcompression(const PGconn *conn); +extern char *PQserverCompression(const PGconn *conn); extern ConnStatusType PQstatus(const PGconn *conn); extern PGTransactionStatusType PQtransactionStatus(const PGconn *conn); extern const char *PQparameterStatus(const PGconn *conn, @@ -501,6 +503,8 @@ extern PGPing PQpingParams(const char *const *keywords, /* Force the write buffer to be written (or at least try) */ extern int PQflush(PGconn *conn); +extern int PQreadPending(PGconn *conn); + /* * "Fast path" interface --- not really recommended for application * use diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 1314663213..4ebd12e420 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -40,6 +40,7 @@ /* include stuff common to fe and be */ #include "libpq/pqcomm.h" +#include "common/zpq_stream.h" /* include stuff found in fe only */ #include "fe-auth-sasl.h" #include "pqexpbuffer.h" @@ -406,6 +407,16 @@ struct pg_conn char *gssdelegation; /* Try to delegate GSS credentials? (0 or 1) */ char *ssl_min_protocol_version; /* minimum TLS protocol version */ char *ssl_max_protocol_version; /* maximum TLS protocol version */ + char *compression; /* stream compression (boolean value, "any" or + * list of compression algorithms separated by + * comma) */ + pg_compress_specification compressors[COMPRESSION_ALGORITHM_COUNT]; /* descriptors of + * compression + * algorithms chosen by + * client */ + unsigned n_compressors; /* size of compressors array */ + char *server_compression; /* compression settings set by server */ + char *target_session_attrs; /* desired session properties */ char *require_auth; /* name of the expected auth method */ char *load_balance_hosts; /* load balance over hosts */ @@ -621,6 +632,9 @@ struct pg_conn /* Buffer for receiving various parts of messages */ PQExpBufferData workBuffer; /* expansible string */ + + /* Compression stream */ + ZpqStream *zpqStream; }; /* PGcancel stores all data necessary to cancel a connection. A copy of this @@ -680,6 +694,7 @@ extern void pqDropConnection(PGconn *conn, bool flushInput); extern int pqPacketSend(PGconn *conn, char pack_type, const void *buf, size_t buf_len); extern bool pqGetHomeDirectory(char *buf, int bufsize); +extern void pqConfigureCompression(PGconn *conn, const char *compressors); extern pgthreadlock_t pg_g_threadlock; @@ -712,7 +727,7 @@ extern void pqParseInput3(PGconn *conn); extern int pqGetErrorNotice3(PGconn *conn, bool isError); extern void pqBuildErrorMessage3(PQExpBuffer msg, const PGresult *res, PGVerbosity verbosity, PGContextVisibility show_context); -extern int pqGetNegotiateProtocolVersion3(PGconn *conn); +extern int pqProcessNegotiateProtocolVersion3(PGconn *conn); extern int pqGetCopyData3(PGconn *conn, char **buffer, int async); extern int pqGetline3(PGconn *conn, char *s, int maxlen); extern int pqGetlineAsync3(PGconn *conn, char *buffer, int bufsize); @@ -750,6 +765,7 @@ extern int pqWaitTimed(int forRead, int forWrite, PGconn *conn, time_t finish_time); extern int pqReadReady(PGconn *conn); extern int pqWriteReady(PGconn *conn); +extern int pqReadPending(PGconn *conn); /* === in fe-secure.c === */ -- 2.42.0 [application/octet-stream] v2-0001-Add-IO-stream-abstraction.patch (79.6K, ../../CACzsqT67iSKp6j+sAZiyE+Jmq12LfzPreYF4-OsgkNk-UoasYA@mail.gmail.com/6-v2-0001-Add-IO-stream-abstraction.patch) download | inline diff: From 4de0c33f161027b242c5ba8d5c43a1c225fb03c6 Mon Sep 17 00:00:00 2001 From: Jacob Burroughs <[email protected]> Date: Tue, 12 Dec 2023 16:28:03 -0600 Subject: [PATCH v2 1/5] Add IO stream abstraction This is a shared abstraction between the frontend and backend that is designed to draw a clean line between the compoments of postgres that want to consume "regular" protocol bytes and any layers that may want to transform them. In this patch, encyryption is simply moved to use this abstraction rather than callers directly using secure_read/secure_write (which depending on context might or might not actually be secure). This is a pre-work change intended to make compression integrate into frontend and backend comms layers more cleanly. This may be a first step towards facilitating greater sharing of SSL/GSS api code between the frontend and backend down the road, but for now it the stream processors themselves are completely independent. --- src/backend/libpq/be-secure-gssapi.c | 63 ++-- src/backend/libpq/be-secure-openssl.c | 75 +++-- src/backend/libpq/be-secure.c | 216 ------------ src/backend/libpq/pqcomm.c | 245 +++++++++++++- src/backend/postmaster/postmaster.c | 5 + src/common/Makefile | 1 + src/common/io_stream.c | 148 ++++++++ src/common/meson.build | 1 + src/include/common/io_stream.h | 131 ++++++++ src/include/libpq/libpq-be.h | 23 +- src/include/libpq/libpq.h | 6 +- src/interfaces/libpq/fe-connect.c | 343 ++++++++++++++++++- src/interfaces/libpq/fe-misc.c | 31 +- src/interfaces/libpq/fe-secure-gssapi.c | 64 ++-- src/interfaces/libpq/fe-secure-openssl.c | 68 +++- src/interfaces/libpq/fe-secure.c | 411 ----------------------- src/interfaces/libpq/libpq-int.h | 42 +-- 17 files changed, 1069 insertions(+), 804 deletions(-) create mode 100644 src/common/io_stream.c create mode 100644 src/include/common/io_stream.h diff --git a/src/backend/libpq/be-secure-gssapi.c b/src/backend/libpq/be-secure-gssapi.c index 8ed2f65347..597ddbae6e 100644 --- a/src/backend/libpq/be-secure-gssapi.c +++ b/src/backend/libpq/be-secure-gssapi.c @@ -74,6 +74,12 @@ static int PqGSSResultNext; /* Next index to read a byte from static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the * results into our output buffer */ +static ssize_t be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +IoStreamProcessor be_gssapi_processor = { + .read = (io_stream_read_func) be_gssapi_read, + .write = (io_stream_write_func) be_gssapi_write +}; /* * Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection. @@ -91,8 +97,8 @@ static uint32 PqGSSMaxPktSize; /* Maximum size we can encrypt and fit the * recursion. Instead, use elog(COMMERROR) to log extra info about the * failure if necessary, and then return an errno indicating connection loss. */ -ssize_t -be_gssapi_write(Port *port, void *ptr, size_t len) +static int +be_gssapi_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written) { OM_uint32 major, minor; @@ -102,6 +108,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) size_t bytes_encrypted; gss_ctx_id_t gctx = port->gss->ctx; + *bytes_written = 0; + /* * When we get a retryable failure, we must not tell the caller we have * successfully transmitted everything, else it won't retry. For @@ -148,20 +156,21 @@ be_gssapi_write(Port *port, void *ptr, size_t len) */ if (PqGSSSendLength) { - ssize_t ret; + int retval; + size_t count; ssize_t amount = PqGSSSendLength - PqGSSSendNext; - ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext, amount); - if (ret <= 0) - return ret; + retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count); + if (retval < 0 || count == 0) + return retval; /* * Check if this was a partial write, and if so, move forward that * far in our buffer and try again. */ - if (ret < amount) + if (count < amount) { - PqGSSSendNext += ret; + PqGSSSendNext += count; continue; } @@ -242,7 +251,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) /* We're reporting all the data as sent, so reset PqGSSSendConsumed. */ PqGSSSendConsumed = 0; - return bytes_encrypted; + *bytes_written = bytes_encrypted; + return 0; } /* @@ -258,8 +268,8 @@ be_gssapi_write(Port *port, void *ptr, size_t len) * We treat fatal errors the same as in be_gssapi_write(), even though the * argument about infinite recursion doesn't apply here. */ -ssize_t -be_gssapi_read(Port *port, void *ptr, size_t len) +static ssize_t +be_gssapi_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) { OM_uint32 major, minor; @@ -269,6 +279,9 @@ be_gssapi_read(Port *port, void *ptr, size_t len) size_t bytes_returned = 0; gss_ctx_id_t gctx = port->gss->ctx; + if (buffered_only) + return false; + /* * The plan here is to read one incoming encrypted packet into * PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out @@ -325,10 +338,10 @@ be_gssapi_read(Port *port, void *ptr, size_t len) /* Collect the length if we haven't already */ if (PqGSSRecvLength < sizeof(uint32)) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, - sizeof(uint32) - PqGSSRecvLength); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + sizeof(uint32) - PqGSSRecvLength, false); - /* If ret <= 0, secure_raw_read already set the correct errno */ + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -359,8 +372,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len) * Read as much of the packet as we are able to on this call into * wherever we left off from the last time we were called. */ - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, - input.length - (PqGSSRecvLength - sizeof(uint32))); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + input.length - (PqGSSRecvLength - sizeof(uint32)), false); /* If ret <= 0, secure_raw_read already set the correct errno */ if (ret <= 0) return ret; @@ -413,7 +426,8 @@ be_gssapi_read(Port *port, void *ptr, size_t len) /* * Read the specified number of bytes off the wire, waiting using - * WaitLatchOrSocket if we would block. + * WaitLatchOrSocket if we would block. Only used during connecition setup + * before GSS is added to the io_stream. * * Results are read into PqGSSRecvBuffer. * @@ -430,7 +444,7 @@ read_or_wait(Port *port, ssize_t len) */ while (PqGSSRecvLength < len) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength); + ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false); /* * If we got back an error and it wasn't just @@ -465,7 +479,7 @@ read_or_wait(Port *port, ssize_t len) */ if (ret == 0) { - ret = secure_raw_read(port, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength); + ret = io_stream_read(port->io_stream, PqGSSRecvBuffer + PqGSSRecvLength, len - PqGSSRecvLength, false); if (ret == 0) return -1; } @@ -648,8 +662,10 @@ secure_open_gssapi(Port *port) while (PqGSSSendNext < PqGSSSendLength) { - ret = secure_raw_write(port, PqGSSSendBuffer + PqGSSSendNext, - PqGSSSendLength - PqGSSSendNext); + size_t count; + + ret = io_stream_write(port->io_stream, PqGSSSendBuffer + PqGSSSendNext, + PqGSSSendLength - PqGSSSendNext, &count); /* * If we got back an error and it wasn't just @@ -663,7 +679,7 @@ secure_open_gssapi(Port *port) } /* Wait and retry if we couldn't write yet */ - if (ret <= 0) + if (ret < 0 || count == 0) { WaitLatchOrSocket(MyLatch, WL_SOCKET_WRITEABLE | WL_EXIT_ON_PM_DEATH, @@ -671,7 +687,7 @@ secure_open_gssapi(Port *port) continue; } - PqGSSSendNext += ret; + PqGSSSendNext += count; } /* Done sending the packet, reset our buffer */ @@ -701,6 +717,7 @@ secure_open_gssapi(Port *port) pg_GSS_error(_("GSSAPI size check error"), major, minor); return -1; } + io_stream_add_layer(port->io_stream, &be_gssapi_processor, port); port->gss->enc = true; diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c index 22e3dc5a81..5c67fd46aa 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c @@ -59,7 +59,7 @@ openssl_tls_init_hook_typ openssl_tls_init_hook = default_openssl_tls_init; static int my_sock_read(BIO *h, char *buf, int size); static int my_sock_write(BIO *h, const char *buf, int size); static BIO_METHOD *my_BIO_s_socket(void); -static int my_SSL_set_fd(Port *port, int fd); +static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer); static DH *load_dh_file(char *filename, bool isServerStart); static DH *load_dh_buffer(const char *buffer, size_t len); @@ -71,6 +71,16 @@ static bool initialize_dh(SSL_CTX *context, bool isServerStart); static bool initialize_ecdh(SSL_CTX *context, bool isServerStart); static const char *SSLerrmessage(unsigned long ecode); +static ssize_t be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +static void be_tls_close(Port *port); + +IoStreamProcessor be_tls_processor = { + .read = (io_stream_read_func) be_tls_read, + .write = (io_stream_write_func) be_tls_write, + .destroy = (io_stream_destroy_func) be_tls_close +}; + static char *X509_NAME_to_cstring(X509_NAME *name); static SSL_CTX *SSL_context = NULL; @@ -440,7 +450,8 @@ be_tls_open_server(Port *port) SSLerrmessage(ERR_get_error())))); return -1; } - if (!my_SSL_set_fd(port, port->sock)) + if (!my_SSL_set_fd(port->ssl, port->sock, + io_stream_add_layer(port->io_stream, &be_tls_processor, port))) { ereport(COMMERROR, (errcode(ERRCODE_PROTOCOL_VIOLATION), @@ -674,7 +685,7 @@ aloop: return 0; } -void +static void be_tls_close(Port *port) { if (port->ssl) @@ -704,14 +715,27 @@ be_tls_close(Port *port) } } -ssize_t -be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) +static ssize_t +be_tls_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) { ssize_t n; int err; unsigned long ecode; + port->waitfor = 0; errno = 0; + + if (buffered_only) + { + /* + * SSL_pending bytes are guaranteed to be available and readable + * without blocking + */ + len = Min(len, SSL_pending(port->ssl)); + if (len == 0) + return 0; + } + ERR_clear_error(); n = SSL_read(port->ssl, ptr, len); err = SSL_get_error(port->ssl, n); @@ -722,12 +746,12 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) /* a-ok */ break; case SSL_ERROR_WANT_READ: - *waitfor = WL_SOCKET_READABLE; + port->waitfor = WL_SOCKET_READABLE; errno = EWOULDBLOCK; n = -1; break; case SSL_ERROR_WANT_WRITE: - *waitfor = WL_SOCKET_WRITEABLE; + port->waitfor = WL_SOCKET_WRITEABLE; errno = EWOULDBLOCK; n = -1; break; @@ -763,13 +787,14 @@ be_tls_read(Port *port, void *ptr, size_t len, int *waitfor) return n; } -ssize_t -be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) +static int +be_tls_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written) { ssize_t n; int err; unsigned long ecode; + port->waitfor = 0; errno = 0; ERR_clear_error(); n = SSL_write(port->ssl, ptr, len); @@ -781,12 +806,12 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) /* a-ok */ break; case SSL_ERROR_WANT_READ: - *waitfor = WL_SOCKET_READABLE; + port->waitfor = WL_SOCKET_READABLE; errno = EWOULDBLOCK; n = -1; break; case SSL_ERROR_WANT_WRITE: - *waitfor = WL_SOCKET_WRITEABLE; + port->waitfor = WL_SOCKET_WRITEABLE; errno = EWOULDBLOCK; n = -1; break; @@ -830,7 +855,16 @@ be_tls_write(Port *port, void *ptr, size_t len, int *waitfor) break; } - return n; + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } } /* ------------------------------------------------------------ */ @@ -858,7 +892,7 @@ my_sock_read(BIO *h, char *buf, int size) if (buf != NULL) { - res = secure_raw_read(((Port *) BIO_get_app_data(h)), buf, size); + res = io_stream_next_read(BIO_get_app_data(h), buf, size, false); BIO_clear_retry_flags(h); if (res <= 0) { @@ -876,11 +910,12 @@ my_sock_read(BIO *h, char *buf, int size) static int my_sock_write(BIO *h, const char *buf, int size) { - int res = 0; + int res; + size_t bytes_written; - res = secure_raw_write(((Port *) BIO_get_app_data(h)), buf, size); + res = io_stream_next_write(BIO_get_app_data(h), buf, size, &bytes_written); BIO_clear_retry_flags(h); - if (res <= 0) + if (res < 0 || bytes_written == 0) { /* If we were interrupted, tell caller to retry */ if (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN) @@ -889,7 +924,7 @@ my_sock_write(BIO *h, const char *buf, int size) } } - return res; + return res ? res : bytes_written; } static BIO_METHOD * @@ -935,7 +970,7 @@ my_BIO_s_socket(void) /* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */ static int -my_SSL_set_fd(Port *port, int fd) +my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer) { int ret = 0; BIO *bio; @@ -954,10 +989,10 @@ my_SSL_set_fd(Port *port, int fd) SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - BIO_set_app_data(bio, port); + BIO_set_app_data(bio, layer); BIO_set_fd(bio, fd, BIO_NOCLOSE); - SSL_set_bio(port->ssl, bio, bio); + SSL_set_bio(ssl, bio, bio); ret = 1; err: return ret; diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c index a0f7084018..caee5b0556 100644 --- a/src/backend/libpq/be-secure.c +++ b/src/backend/libpq/be-secure.c @@ -125,219 +125,3 @@ secure_open_server(Port *port) return r; } - -/* - * Close secure session. - */ -void -secure_close(Port *port) -{ -#ifdef USE_SSL - if (port->ssl_in_use) - be_tls_close(port); -#endif -} - -/* - * Read data from a secure connection. - */ -ssize_t -secure_read(Port *port, void *ptr, size_t len) -{ - ssize_t n; - int waitfor; - - /* Deal with any already-pending interrupt condition. */ - ProcessClientReadInterrupt(false); - -retry: -#ifdef USE_SSL - waitfor = 0; - if (port->ssl_in_use) - { - n = be_tls_read(port, ptr, len, &waitfor); - } - else -#endif -#ifdef ENABLE_GSS - if (port->gss && port->gss->enc) - { - n = be_gssapi_read(port, ptr, len); - waitfor = WL_SOCKET_READABLE; - } - else -#endif - { - n = secure_raw_read(port, ptr, len); - waitfor = WL_SOCKET_READABLE; - } - - /* In blocking mode, wait until the socket is ready */ - if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) - { - WaitEvent event; - - Assert(waitfor); - - ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL); - - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, - WAIT_EVENT_CLIENT_READ); - - /* - * If the postmaster has died, it's not safe to continue running, - * because it is the postmaster's job to kill us if some other backend - * exits uncleanly. Moreover, we won't run very well in this state; - * helper processes like walwriter and the bgwriter will exit, so - * performance may be poor. Finally, if we don't exit, pg_ctl will be - * unable to restart the postmaster without manual intervention, so no - * new connections can be accepted. Exiting clears the deck for a - * postmaster restart. - * - * (Note that we only make this check when we would otherwise sleep on - * our latch. We might still continue running for a while if the - * postmaster is killed in mid-query, or even through multiple queries - * if we never have to wait for read. We don't want to burn too many - * cycles checking for this very rare condition, and this should cause - * us to exit quickly in most cases.) - */ - if (event.events & WL_POSTMASTER_DEATH) - ereport(FATAL, - (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("terminating connection due to unexpected postmaster exit"))); - - /* Handle interrupt. */ - if (event.events & WL_LATCH_SET) - { - ResetLatch(MyLatch); - ProcessClientReadInterrupt(true); - - /* - * We'll retry the read. Most likely it will return immediately - * because there's still no data available, and we'll wait for the - * socket to become ready again. - */ - } - goto retry; - } - - /* - * Process interrupts that happened during a successful (or non-blocking, - * or hard-failed) read. - */ - ProcessClientReadInterrupt(false); - - return n; -} - -ssize_t -secure_raw_read(Port *port, void *ptr, size_t len) -{ - ssize_t n; - - /* - * Try to read from the socket without blocking. If it succeeds we're - * done, otherwise we'll wait for the socket using the latch mechanism. - */ -#ifdef WIN32 - pgwin32_noblock = true; -#endif - n = recv(port->sock, ptr, len, 0); -#ifdef WIN32 - pgwin32_noblock = false; -#endif - - return n; -} - - -/* - * Write data to a secure connection. - */ -ssize_t -secure_write(Port *port, void *ptr, size_t len) -{ - ssize_t n; - int waitfor; - - /* Deal with any already-pending interrupt condition. */ - ProcessClientWriteInterrupt(false); - -retry: - waitfor = 0; -#ifdef USE_SSL - if (port->ssl_in_use) - { - n = be_tls_write(port, ptr, len, &waitfor); - } - else -#endif -#ifdef ENABLE_GSS - if (port->gss && port->gss->enc) - { - n = be_gssapi_write(port, ptr, len); - waitfor = WL_SOCKET_WRITEABLE; - } - else -#endif - { - n = secure_raw_write(port, ptr, len); - waitfor = WL_SOCKET_WRITEABLE; - } - - if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) - { - WaitEvent event; - - Assert(waitfor); - - ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, waitfor, NULL); - - WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, - WAIT_EVENT_CLIENT_WRITE); - - /* See comments in secure_read. */ - if (event.events & WL_POSTMASTER_DEATH) - ereport(FATAL, - (errcode(ERRCODE_ADMIN_SHUTDOWN), - errmsg("terminating connection due to unexpected postmaster exit"))); - - /* Handle interrupt. */ - if (event.events & WL_LATCH_SET) - { - ResetLatch(MyLatch); - ProcessClientWriteInterrupt(true); - - /* - * We'll retry the write. Most likely it will return immediately - * because there's still no buffer space available, and we'll wait - * for the socket to become ready again. - */ - } - goto retry; - } - - /* - * Process interrupts that happened during a successful (or non-blocking, - * or hard-failed) write. - */ - ProcessClientWriteInterrupt(false); - - return n; -} - -ssize_t -secure_raw_write(Port *port, const void *ptr, size_t len) -{ - ssize_t n; - -#ifdef WIN32 - pgwin32_noblock = true; -#endif - n = send(port->sock, ptr, len, 0); -#ifdef WIN32 - pgwin32_noblock = false; -#endif - - return n; -} diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 0084a9bf13..030686cc3b 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -31,6 +31,7 @@ * setup/teardown: * StreamServerPort - Open postmaster's server port * StreamConnection - Create new connection with client + * StreamSetupIo - Configures IO stream on connection * StreamClose - Close a client/backend connection * TouchSocketFiles - Protect socket files against /tmp cleaners * pq_init - initialize libpq at backend startup @@ -74,12 +75,15 @@ #endif #include "common/ip.h" +#include "common/io_stream.h" #include "libpq/libpq.h" #include "miscadmin.h" #include "port/pg_bswap.h" #include "storage/ipc.h" +#include "tcop/tcopprot.h" #include "utils/guc_hooks.h" #include "utils/memutils.h" +#include "utils/wait_event.h" /* * Cope with the various platform-specific ways to spell TCP keepalive socket @@ -146,6 +150,15 @@ static int socket_putmessage(char msgtype, const char *s, size_t len); static void socket_putmessage_noblock(char msgtype, const char *s, size_t len); static int internal_putbytes(const char *s, size_t len); static int internal_flush(void); +static ssize_t socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only); +static int socket_write(IoStreamLayer * self, Port *port, void const *ptr, size_t len, size_t *bytes_written); +static ssize_t io_read_with_wait(Port *port, void *ptr, size_t len); +static int io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_written); + +IoStreamProcessor socket_processor = { + .read = (io_stream_read_func) socket_read, + .write = (io_stream_write_func) socket_write +}; static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); static int Setup_AF_UNIX(const char *sock_path); @@ -277,7 +290,8 @@ socket_close(int code, Datum arg) * Cleanly shut down SSL layer. Nowhere else does a postmaster child * call this, so this is safe when interrupting BackendInitialize(). */ - secure_close(MyProcPort); + io_stream_destroy(MyProcPort->io_stream); + MyProcPort->io_stream = NULL; /* * Formerly we did an explicit close() here, but it seems better to @@ -817,6 +831,30 @@ StreamConnection(pgsocket server_fd, Port *port) return STATUS_OK; } +/* This must be called after the child process is launched or the data structures + * do not comre across correctly + */ +int +StreamSetupIo(Port *port) +{ + if (port->io_stream) + { + ereport(LOG, + (errmsg("%s() failed: io_stream already configured", "io_stream_create"))); + return STATUS_ERROR; + } + port->io_stream = io_stream_create(); + if (!port->io_stream) + { + ereport(LOG, + (errmsg("%s() failed: %m", "io_stream_create"))); + return STATUS_ERROR; + } + io_stream_add_layer(port->io_stream, &socket_processor, port); + + return STATUS_OK; +} + /* * StreamClose -- close a client/backend connection * @@ -905,6 +943,193 @@ socket_set_nonblocking(bool nonblocking) MyProcPort->noblock = nonblocking; } +static ssize_t +socket_read(IoStreamLayer * self, Port *port, void *ptr, size_t len, bool buffered_only) +{ + ssize_t n; + + if (buffered_only) + return 0; + + /* + * Try to read from the socket without blocking. If it succeeds we're + * done, otherwise we'll wait for the socket using the latch mechanism. + */ +#ifdef WIN32 + pgwin32_noblock = true; +#endif + n = recv(port->sock, ptr, len, 0); +#ifdef WIN32 + pgwin32_noblock = false; +#endif + + return n; +} + +static int +socket_write(IoStreamLayer * self, Port *port, const void *ptr, size_t len, size_t *bytes_written) +{ + ssize_t n; + +#ifdef WIN32 + pgwin32_noblock = true; +#endif + n = send(port->sock, ptr, len, 0); +#ifdef WIN32 + pgwin32_noblock = false; +#endif + + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } +} + +/* + * Read protocol-level data, processed through any intermediate streams like TLS + */ +static ssize_t +io_read_with_wait(Port *port, void *ptr, size_t len) +{ + ssize_t n; + + /* Deal with any already-pending interrupt condition. */ + ProcessClientReadInterrupt(false); + +retry: + port->waitfor = WL_SOCKET_READABLE; + n = io_stream_read(port->io_stream, ptr, len, false); + + /* In blocking mode, wait until the socket is ready */ + if (n < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) + { + WaitEvent event; + + Assert(port->waitfor); + + ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL); + + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_READ); + + /* + * If the postmaster has died, it's not safe to continue running, + * because it is the postmaster's job to kill us if some other backend + * exits uncleanly. Moreover, we won't run very well in this state; + * helper processes like walwriter and the bgwriter will exit, so + * performance may be poor. Finally, if we don't exit, pg_ctl will be + * unable to restart the postmaster without manual intervention, so no + * new connections can be accepted. Exiting clears the deck for a + * postmaster restart. + * + * (Note that we only make this check when we would otherwise sleep on + * our latch. We might still continue running for a while if the + * postmaster is killed in mid-query, or even through multiple queries + * if we never have to wait for read. We don't want to burn too many + * cycles checking for this very rare condition, and this should cause + * us to exit quickly in most cases.) + */ + if (event.events & WL_POSTMASTER_DEATH) + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating connection due to unexpected postmaster exit"))); + + /* Handle interrupt. */ + if (event.events & WL_LATCH_SET) + { + ResetLatch(MyLatch); + ProcessClientReadInterrupt(true); + + /* + * We'll retry the read. Most likely it will return immediately + * because there's still no data available, and we'll wait for the + * socket to become ready again. + */ + } + goto retry; + } + + /* + * Process interrupts that happened during a successful (or non-blocking, + * or hard-failed) read. + */ + ProcessClientReadInterrupt(false); + + return n; +} + + +/* + * Write protocol-level data to be processed through any intermediate streams like TLS + */ +static int +io_write_with_wait(Port *port, char const *ptr, size_t len, size_t *bytes_processed) +{ + int rc; + size_t count = 0; + + *bytes_processed = 0; + + /* Deal with any already-pending interrupt condition. */ + ProcessClientWriteInterrupt(false); + +retry: + port->waitfor = WL_SOCKET_WRITEABLE; + + /* + * on retry it is possible some of the input will have already been + * processed, so make sure we offset our retries + */ + rc = io_stream_write(port->io_stream, ptr + *bytes_processed, len - *bytes_processed, &count); + *bytes_processed += count; + + if (rc < 0 && !port->noblock && (errno == EWOULDBLOCK || errno == EAGAIN)) + { + WaitEvent event; + + Assert(port->waitfor); + + ModifyWaitEvent(FeBeWaitSet, FeBeWaitSetSocketPos, port->waitfor, NULL); + + WaitEventSetWait(FeBeWaitSet, -1 /* no timeout */ , &event, 1, + WAIT_EVENT_CLIENT_WRITE); + + /* See comments in secure_read. */ + if (event.events & WL_POSTMASTER_DEATH) + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating connection due to unexpected postmaster exit"))); + + /* Handle interrupt. */ + if (event.events & WL_LATCH_SET) + { + ResetLatch(MyLatch); + ProcessClientWriteInterrupt(true); + + /* + * We'll retry the write. Most likely it will return immediately + * because there's still no buffer space available, and we'll wait + * for the socket to become ready again. + */ + } + goto retry; + } + + /* + * Process interrupts that happened during a successful (or non-blocking, + * or hard-failed) write. + */ + ProcessClientWriteInterrupt(false); + + return rc; +} + /* -------------------------------- * pq_recvbuf - load some bytes into the input buffer * @@ -938,8 +1163,8 @@ pq_recvbuf(void) errno = 0; - r = secure_read(MyProcPort, PqRecvBuffer + PqRecvLength, - PQ_RECV_BUFFER_SIZE - PqRecvLength); + r = io_read_with_wait(MyProcPort, PqRecvBuffer + PqRecvLength, + PQ_RECV_BUFFER_SIZE - PqRecvLength); if (r < 0) { @@ -1035,7 +1260,7 @@ pq_getbyte_if_available(unsigned char *c) errno = 0; - r = secure_read(MyProcPort, c, 1); + r = io_read_with_wait(MyProcPort, c, 1); if (r < 0) { /* @@ -1351,11 +1576,15 @@ internal_flush(void) while (bufptr < bufend) { - int r; + int rc; + size_t bytes_sent; + size_t available = bufend - bufptr; - r = secure_write(MyProcPort, bufptr, bufend - bufptr); + rc = io_write_with_wait(MyProcPort, bufptr, available, &bytes_sent); + bufptr += bytes_sent; + PqSendStart += bytes_sent; - if (r <= 0) + if (rc < 0 || (bytes_sent == 0 && available)) { if (errno == EINTR) continue; /* Ok if we were interrupted */ @@ -1400,8 +1629,6 @@ internal_flush(void) } last_reported_send_errno = 0; /* reset after any successful send */ - bufptr += r; - PqSendStart += r; } PqSendStart = PqSendPointer = 0; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index b163e89cbb..1f94e2910c 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -4232,6 +4232,11 @@ BackendInitialize(Port *port) /* Save port etc. for ps status */ MyProcPort = port; + if (StreamSetupIo(port)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("Unable to configure backend I/O"))); + /* Tell fd.c about the long-lived FD associated with the port */ ReserveExternalFD(); diff --git a/src/common/Makefile b/src/common/Makefile index 2ba5069dca..a80f4b39a6 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -59,6 +59,7 @@ OBJS_COMMON = \ file_perm.o \ file_utils.o \ hashfn.o \ + io_stream.o \ ip.o \ jsonapi.o \ keywords.o \ diff --git a/src/common/io_stream.c b/src/common/io_stream.c new file mode 100644 index 0000000000..b15aca326d --- /dev/null +++ b/src/common/io_stream.c @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * io_stream.c + * functions related to managing layers of streaming IO. + * In general the base layer will work with raw sockets, and then + * additional layers will add features such as encryption and + * compression. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/common/io_stream.c + * + *------------------------------------------------------------------------- + */ + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include <common/io_stream.h> + +#ifndef FRONTEND +#include "utils/memutils.h" +#define ALLOC(size) MemoryContextAlloc(TopMemoryContext, size) +#define FREE(size) pfree(size) +#else +#define ALLOC(size) malloc(size) +#define FREE(size) free(size) +#endif + +struct IoStreamLayer +{ + IoStreamProcessor *processor; + void *context; + IoStreamLayer *next; +}; + +struct IoStream +{ + IoStreamLayer *layer; +}; + +IoStream * +io_stream_create(void) +{ + IoStream *ret = ALLOC(sizeof(IoStream)); + + ret->layer = NULL; + return ret; +} + +void +io_stream_destroy(IoStream * arg) +{ + IoStreamLayer *layer; + + if (arg == NULL) + return; + + layer = arg->layer; + while (layer != NULL) + { + IoStreamLayer *next = layer->next; + + if (layer->processor->destroy != NULL) + layer->processor->destroy(layer->context); + FREE(layer); + layer = next; + } + FREE(arg); +} + +IoStreamLayer * +io_stream_add_layer(IoStream * stream, IoStreamProcessor * processor, void *context) +{ + IoStreamLayer *layer = ALLOC(sizeof(IoStreamLayer)); + + layer->processor = processor; + layer->context = context; + layer->next = stream->layer; + stream->layer = layer; + return layer; +} + +ssize_t +io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only) +{ + if (stream->layer == NULL) + return -1; + + return stream->layer->processor->read(stream->layer, stream->layer->context, data, size, buffered_only); +} + +int +io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written) +{ + if (stream->layer == NULL) + return -1; + + return stream->layer->processor->write(stream->layer, stream->layer->context, data, size, bytes_written); +} + +bool +io_stream_buffered_read_data(IoStream * stream) +{ + IoStreamLayer *layer; + + for (layer = stream->layer; layer != NULL; layer = layer->next) + { + if (layer->processor->buffered_read_data != NULL && layer->processor->buffered_read_data(layer->context)) + return true; + } + return false; +} + +bool +io_stream_buffered_write_data(IoStream * stream) +{ + IoStreamLayer *layer; + + for (layer = stream->layer; layer != NULL; layer = layer->next) + { + if (layer->processor->buffered_write_data != NULL && layer->processor->buffered_write_data(layer->context)) + return true; + } + return false; +} + +ssize_t +io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only) +{ + if (layer->next == NULL) + return -1; + + return layer->next->processor->read(layer->next, layer->next->context, data, size, buffered_only); +} + +int +io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written) +{ + if (layer->next == NULL) + return -1; + + return layer->next->processor->write(layer->next, layer->next->context, data, size, bytes_written); +} diff --git a/src/common/meson.build b/src/common/meson.build index 12fd43e87f..e25fa44133 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -13,6 +13,7 @@ common_sources = files( 'file_perm.c', 'file_utils.c', 'hashfn.c', + 'io_stream.c', 'ip.c', 'jsonapi.c', 'keywords.c', diff --git a/src/include/common/io_stream.h b/src/include/common/io_stream.h new file mode 100644 index 0000000000..9af88aa9ea --- /dev/null +++ b/src/include/common/io_stream.h @@ -0,0 +1,131 @@ +/*------------------------------------------------------------------------- + * + * io_stream.c + * defintions for managing layers of streaming IO. + * In general the base layer will work with raw sockets, and then + * additional layers will add features such as encryption and + * compression. + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/include/common/io_stream.h + * + *------------------------------------------------------------------------- + */ +#ifndef IO_STREAM_H +#define IO_STREAM_H + +/* Opaque structs should only be interacted with through corresponding functions*/ +typedef struct IoStreamLayer IoStreamLayer; +typedef struct IoStream IoStream; + +typedef ssize_t (*io_stream_read_func) (IoStreamLayer * self, void *context, void *data, size_t size, bool buffered_only); +typedef int (*io_stream_write_func) (IoStreamLayer * self, void *context, void const *data, size_t size, size_t *bytes_written); +typedef bool (*io_stream_predicate) (void *context); +typedef void (*io_stream_destroy_func) (void *context); + +typedef struct IoStreamProcessor +{ + /* + * Required Should call io_stream_next_read with self either directly or + * indirectly to recieve bytes from the underlying layer of the stream + */ + io_stream_read_func read; + + /* + * Required Should call io_stream_next_write with self either directly or + * indirectly to write bytes to the underlying layer of the stream + */ + io_stream_write_func write; + + /* + * Optional Return true if this layer is holding buffered readable data + */ + io_stream_predicate buffered_read_data; + + /* + * Optional Return true if this layer is holding buffered writable data + */ + io_stream_predicate buffered_write_data; + + /* + * Optional will be called as part of io_stream_destroy when cleaning up + * the stream + */ + io_stream_destroy_func destroy; +} IoStreamProcessor; + +/* + * Allocate a new IoStream and return the address to the caller. IoStreams should always be destroyed with + * the corresponding io_stream_destroy function + */ +extern IoStream * io_stream_create(void); +/* + * Adds new processors to the IO stream + * + * processorDescription contains collection of function pointers for this layer + * context (optional) pointer with context to be used in reader/writer + * + * Returns the newly created processor + * */ +extern IoStreamLayer * io_stream_add_layer(IoStream * stream, IoStreamProcessor * processorDescription, void *context); +/* + * Destroy an IoStream, freeing all associated memory + */ +extern void io_stream_destroy(IoStream * stream); + +/* + * Read data from the stream + * + * Reads at most size bytes into the buffer pointed to by data, and returns + * the number of bytes read. If buffered_only is true, then only data that + * was stored in an in-process buffer will be returned and this function will + * not block. In that case, a return value of 0 simply means there was no + * buffered data available and does not mean the stream has reached EOF. + */ +extern ssize_t io_stream_read(IoStream * stream, void *data, size_t size, bool buffered_only); + +/* + * Write data to the stream + * + * Writes at most size bytes from the buffer pointed to by data, and returns + * true on success, false on failure, with the specific error in errno. The + * count of bytes written from data will be stored in bytes_written and may + * be non-zero even on failure + */ +extern int io_stream_write(IoStream * stream, void const *data, size_t size, size_t *bytes_written); + +/* + * Check if there is data buffered in memory waiting to be read (e.g. if + * compression is in use and more uncompressed data was read than fit into + * the provided buffer) + */ +extern bool io_stream_buffered_read_data(IoStream * stream); + +/* + * Check if there is data buffered in memory waiting to be written to the underlying backing store + */ +extern bool io_stream_buffered_write_data(IoStream * stream); + +/* + * Read data from the next layer of the stream + * (to be used by io_stream_read_func) + * + * Reads at most size bytes into the buffer pointed to by data, and returns + * the number of bytes read + */ +extern ssize_t io_stream_next_read(IoStreamLayer * layer, void *data, size_t size, bool buffered_only); + +/* + * Write data to the next layer of the stream + * (to be used by io_stream_write_func) + * + * Writes at most size bytes from the buffer pointed to by data, and returns + * true on success, false on failure, with the specific error in errno. The + * count of bytes written from data will be stored in bytes_written and may + * be non-zero even on failure + */ +extern int io_stream_next_write(IoStreamLayer * layer, void const *data, size_t size, size_t *bytes_written); + +#endif /* //IO_STREAM_H */ diff --git a/src/include/libpq/libpq-be.h b/src/include/libpq/libpq-be.h index c57ed12fb6..87ba7f5ea0 100644 --- a/src/include/libpq/libpq-be.h +++ b/src/include/libpq/libpq-be.h @@ -53,6 +53,7 @@ typedef struct #endif #endif /* ENABLE_SSPI */ +#include "common/io_stream.h" #include "datatype/timestamp.h" #include "libpq/hba.h" #include "libpq/pqcomm.h" @@ -145,8 +146,11 @@ typedef struct ClientConnectionInfo typedef struct Port { + IoStream *io_stream; pgsocket sock; /* File descriptor */ bool noblock; /* is the socket in non-blocking mode? */ + int waitfor; /* Events to wait on the socket for after + * attempted read/write */ ProtocolVersion proto; /* FE/BE protocol version */ SockAddr laddr; /* local addr (postmaster) */ SockAddr raddr; /* remote addr (client) */ @@ -274,21 +278,6 @@ extern void be_tls_destroy(void); */ extern int be_tls_open_server(Port *port); -/* - * Close SSL connection. - */ -extern void be_tls_close(Port *port); - -/* - * Read data from a secure connection. - */ -extern ssize_t be_tls_read(Port *port, void *ptr, size_t len, int *waitfor); - -/* - * Write data to a secure connection. - */ -extern ssize_t be_tls_write(Port *port, void *ptr, size_t len, int *waitfor); - /* * Return information about the SSL connection. */ @@ -324,10 +313,6 @@ extern bool be_gssapi_get_auth(Port *port); extern bool be_gssapi_get_enc(Port *port); extern const char *be_gssapi_get_princ(Port *port); extern bool be_gssapi_get_delegation(Port *port); - -/* Read and write to a GSSAPI-encrypted connection. */ -extern ssize_t be_gssapi_read(Port *port, void *ptr, size_t len); -extern ssize_t be_gssapi_write(Port *port, void *ptr, size_t len); #endif /* ENABLE_GSS */ extern PGDLLIMPORT ProtocolVersion FrontendProtocol; diff --git a/src/include/libpq/libpq.h b/src/include/libpq/libpq.h index a6104d8cd0..3612280146 100644 --- a/src/include/libpq/libpq.h +++ b/src/include/libpq/libpq.h @@ -68,6 +68,7 @@ extern int StreamServerPort(int family, const char *hostName, unsigned short portNumber, const char *unixSocketDir, pgsocket ListenSocket[], int *NumListenSockets, int MaxListen); extern int StreamConnection(pgsocket server_fd, Port *port); +extern int StreamSetupIo(Port *port); extern void StreamClose(pgsocket sock); extern void TouchSocketFiles(void); extern void RemoveSocketFiles(void); @@ -104,11 +105,6 @@ extern int secure_initialize(bool isServerStart); extern bool secure_loaded_verify_locations(void); extern void secure_destroy(void); extern int secure_open_server(Port *port); -extern void secure_close(Port *port); -extern ssize_t secure_read(Port *port, void *ptr, size_t len); -extern ssize_t secure_write(Port *port, void *ptr, size_t len); -extern ssize_t secure_raw_read(Port *port, void *ptr, size_t len); -extern ssize_t secure_raw_write(Port *port, const void *ptr, size_t len); /* * prototypes for functions in be-secure-gssapi.c diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index bf83a9b569..a0f12e62af 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -136,6 +136,55 @@ static int ldapServiceLookup(const char *purl, PQconninfoOption *options, #define DefaultGSSMode "disable" #endif +/* + * Macros to handle disabling and then restoring the state of SIGPIPE handling. + * On Windows, these are all no-ops since there's no SIGPIPEs. + */ + +#ifndef WIN32 + +#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag) + +struct sigpipe_info +{ + sigset_t oldsigmask; + bool sigpipe_pending; + bool got_epipe; +}; + +#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo + +#define DISABLE_SIGPIPE(conn, spinfo, failaction) \ +do { \ +(spinfo).got_epipe = false; \ +if (!SIGPIPE_MASKED(conn)) \ +{ \ +if (pq_block_sigpipe(&(spinfo).oldsigmask, \ +&(spinfo).sigpipe_pending) < 0) \ +failaction; \ +} \ +} while (0) + +#define REMEMBER_EPIPE(spinfo, cond) \ +do { \ +if (cond) \ +(spinfo).got_epipe = true; \ +} while (0) + +#define RESTORE_SIGPIPE(conn, spinfo) \ +do { \ +if (!SIGPIPE_MASKED(conn)) \ +pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \ +(spinfo).got_epipe); \ +} while (0) +#else /* WIN32 */ + +#define DECLARE_SIGPIPE_INFO(spinfo) +#define DISABLE_SIGPIPE(conn, spinfo, failaction) +#define REMEMBER_EPIPE(spinfo, cond) +#define RESTORE_SIGPIPE(conn, spinfo) +#endif /* WIN32 */ + /* ---------- * Definition of the conninfo parameters and their fallback resources. * @@ -445,6 +494,12 @@ static bool sslVerifyProtocolVersion(const char *version); static bool sslVerifyProtocolRange(const char *min, const char *max); static bool parse_int_param(const char *value, int *result, PGconn *conn, const char *context); +static ssize_t socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written); +IoStreamProcessor socket_processor = { + .read = (io_stream_read_func) socket_read, + .write = (io_stream_write_func) socket_write +}; /* global variable because fe-auth.c needs to access it */ @@ -466,9 +521,8 @@ pgthreadlock_t pg_g_threadlock = default_threadlock; void pqDropConnection(PGconn *conn, bool flushInput) { - /* Drop any SSL state */ - pqsecure_close(conn); - + io_stream_destroy(conn->io_stream); + conn->io_stream = NULL; /* Close the socket itself */ if (conn->sock != PGINVALID_SOCKET) closesocket(conn->sock); @@ -2879,6 +2933,9 @@ keep_going: /* We will come back to here until there is sock_type |= SOCK_NONBLOCK; #endif conn->sock = socket(addr_cur->family, sock_type, 0); + conn->io_stream = io_stream_create(); + io_stream_add_layer(conn->io_stream, &socket_processor, conn); + if (conn->sock == PGINVALID_SOCKET) { int errorno = SOCK_ERRNO; @@ -7829,3 +7886,283 @@ PQregisterThreadLock(pgthreadlock_t newhandler) return prev; } + +static ssize_t +socket_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) +{ + ssize_t n; + int result_errno = 0; + char sebuf[PG_STRERROR_R_BUFLEN]; + + SOCK_ERRNO_SET(0); + + if (buffered_only) + return 0; + + n = recv(conn->sock, ptr, len, 0); + + if (n < 0) + { + result_errno = SOCK_ERRNO; + + /* Set error message if appropriate */ + switch (result_errno) + { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) + case EWOULDBLOCK: +#endif + case EINTR: + /* no error message, caller is expected to retry */ + break; + + case EPIPE: + case ECONNRESET: + libpq_append_conn_error(conn, "server closed the connection unexpectedly\n" + "\tThis probably means the server terminated abnormally\n" + "\tbefore or while processing the request."); + break; + + case 0: + /* If errno didn't get set, treat it as regular EOF */ + n = 0; + break; + + default: + libpq_append_conn_error(conn, "could not receive data from server: %s", + SOCK_STRERROR(result_errno, + sebuf, sizeof(sebuf))); + break; + } + } + + /* ensure we return the intended errno to caller */ + SOCK_ERRNO_SET(result_errno); + + return n; +} + +/* + * Socket-level implementation of data writing. + * + * This is used directly for an unencrypted connection. For encrypted + * connections, this is wrapped by higher layers through IO stream + * + * This function reports failure (i.e., returns a negative result) only + * for retryable errors such as EINTR. Looping for such cases is to be + * handled at some outer level, maybe all the way up to the application. + * For hard failures, we set conn->write_failed and store an error message + * in conn->write_err_msg, but then claim to have written the data anyway. + * This is because we don't want to report write failures so long as there + * is a possibility of reading from the server and getting an error message + * that could explain why the connection dropped. Many TCP stacks have + * race conditions such that a write failure may or may not be reported + * before all incoming data has been read. + * + * Note that this error behavior happens below the SSL management level when + * we are using SSL. That's because at least some versions of OpenSSL are + * too quick to report a write failure when there's still a possibility to + * get a more useful error from the server. + */ +static int +socket_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written) +{ + ssize_t n; + int flags = 0; + int result_errno = 0; + char msgbuf[1024]; + char sebuf[PG_STRERROR_R_BUFLEN]; + + DECLARE_SIGPIPE_INFO(spinfo); + + /* + * If we already had a write failure, we will never again try to send data + * on that connection. Even if the kernel would let us, we've probably + * lost message boundary sync with the server. conn->write_failed + * therefore persists until the connection is reset, and we just discard + * all data presented to be written. + */ + if (conn->write_failed) + return len; + +#ifdef MSG_NOSIGNAL + if (conn->sigpipe_flag) + flags |= MSG_NOSIGNAL; + +retry_masked: +#endif /* MSG_NOSIGNAL */ + + DISABLE_SIGPIPE(conn, spinfo, return -1); + + n = send(conn->sock, ptr, len, flags); + + if (n < 0) + { + *bytes_written = 0; + result_errno = SOCK_ERRNO; + + /* + * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available + * on this machine. So, clear sigpipe_flag so we don't try the flag + * again, and retry the send(). + */ +#ifdef MSG_NOSIGNAL + if (flags != 0 && result_errno == EINVAL) + { + conn->sigpipe_flag = false; + flags = 0; + goto retry_masked; + } +#endif /* MSG_NOSIGNAL */ + + /* Set error message if appropriate */ + switch (result_errno) + { +#ifdef EAGAIN + case EAGAIN: +#endif +#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) + case EWOULDBLOCK: +#endif + case EINTR: + /* no error message, caller is expected to retry */ + break; + + case EPIPE: + /* Set flag for EPIPE */ + REMEMBER_EPIPE(spinfo, true); + + /* FALL THRU */ + + case ECONNRESET: + conn->write_failed = true; + /* Store error message in conn->write_err_msg, if possible */ + /* (strdup failure is OK, we'll cope later) */ + snprintf(msgbuf, sizeof(msgbuf), + libpq_gettext("server closed the connection unexpectedly\n" + "\tThis probably means the server terminated abnormally\n" + "\tbefore or while processing the request.")); + /* keep newline out of translated string */ + strlcat(msgbuf, "\n", sizeof(msgbuf)); + conn->write_err_msg = strdup(msgbuf); + /* Now claim the write succeeded */ + n = len; + break; + + default: + conn->write_failed = true; + /* Store error message in conn->write_err_msg, if possible */ + /* (strdup failure is OK, we'll cope later) */ + snprintf(msgbuf, sizeof(msgbuf), + libpq_gettext("could not send data to server: %s"), + SOCK_STRERROR(result_errno, + sebuf, sizeof(sebuf))); + /* keep newline out of translated string */ + strlcat(msgbuf, "\n", sizeof(msgbuf)); + conn->write_err_msg = strdup(msgbuf); + /* Now claim the write succeeded */ + n = len; + break; + } + } + else + { + *bytes_written = n; + n = 0; + } + + RESTORE_SIGPIPE(conn, spinfo); + + /* ensure we return the intended errno to caller */ + SOCK_ERRNO_SET(result_errno); + + return n; +} + +#if !defined(WIN32) + +/* + * Block SIGPIPE for this thread. This prevents send()/write() from exiting + * the application. + */ +int +pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending) +{ + sigset_t sigpipe_sigset; + sigset_t sigset; + + sigemptyset(&sigpipe_sigset); + sigaddset(&sigpipe_sigset, SIGPIPE); + + /* Block SIGPIPE and save previous mask for later reset */ + SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset)); + if (SOCK_ERRNO) + return -1; + + /* We can have a pending SIGPIPE only if it was blocked before */ + if (sigismember(osigset, SIGPIPE)) + { + /* Is there a pending SIGPIPE? */ + if (sigpending(&sigset) != 0) + return -1; + + if (sigismember(&sigset, SIGPIPE)) + *sigpipe_pending = true; + else + *sigpipe_pending = false; + } + else + *sigpipe_pending = false; + + return 0; +} + +/* + * Discard any pending SIGPIPE and reset the signal mask. + * + * Note: we are effectively assuming here that the C library doesn't queue + * up multiple SIGPIPE events. If it did, then we'd accidentally leave + * ours in the queue when an event was already pending and we got another. + * As long as it doesn't queue multiple events, we're OK because the caller + * can't tell the difference. + * + * The caller should say got_epipe = false if it is certain that it + * didn't get an EPIPE error; in that case we'll skip the clear operation + * and things are definitely OK, queuing or no. If it got one or might have + * gotten one, pass got_epipe = true. + * + * We do not want this to change errno, since if it did that could lose + * the error code from a preceding send(). We essentially assume that if + * we were able to do pq_block_sigpipe(), this can't fail. + */ +void +pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe) +{ + int save_errno = SOCK_ERRNO; + int signo; + sigset_t sigset; + + /* Clear SIGPIPE only if none was pending */ + if (got_epipe && !sigpipe_pending) + { + if (sigpending(&sigset) == 0 && + sigismember(&sigset, SIGPIPE)) + { + sigset_t sigpipe_sigset; + + sigemptyset(&sigpipe_sigset); + sigaddset(&sigpipe_sigset, SIGPIPE); + + sigwait(&sigpipe_sigset, &signo); + } + } + + /* Restore saved block mask */ + pthread_sigmask(SIG_SETMASK, osigset, NULL); + + SOCK_ERRNO_SET(save_errno); +} + +#endif /* !WIN32 */ diff --git a/src/interfaces/libpq/fe-misc.c b/src/interfaces/libpq/fe-misc.c index 660cdec93c..6772f2876d 100644 --- a/src/interfaces/libpq/fe-misc.c +++ b/src/interfaces/libpq/fe-misc.c @@ -614,8 +614,8 @@ pqReadData(PGconn *conn) /* OK, try to read some data */ retry3: - nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, - conn->inBufSize - conn->inEnd); + nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, + conn->inBufSize - conn->inEnd, false); if (nread < 0) { switch (SOCK_ERRNO) @@ -709,8 +709,8 @@ retry3: * arrived. */ retry4: - nread = pqsecure_read(conn, conn->inBuffer + conn->inEnd, - conn->inBufSize - conn->inEnd); + nread = io_stream_read(conn->io_stream, conn->inBuffer + conn->inEnd, + conn->inBufSize - conn->inEnd, false); if (nread < 0) { switch (SOCK_ERRNO) @@ -824,10 +824,11 @@ pqSendSome(PGconn *conn, int len) /* while there's still data to send */ while (len > 0) { - int sent; + size_t sent; + int rc; #ifndef WIN32 - sent = pqsecure_write(conn, ptr, len); + rc = io_stream_write(conn->io_stream, ptr, len, &sent); #else /* @@ -835,10 +836,13 @@ pqSendSome(PGconn *conn, int len) * failure-point appears to be different in different versions of * Windows, but 64k should always be safe. */ - sent = pqsecure_write(conn, ptr, Min(len, 65536)); + rc = io_stream_write(conn->io_stream, ptr, Min(len, 65536), &sent); #endif + ptr += sent; + len -= sent; + remaining -= sent; - if (sent < 0) + if (rc < 0) { /* Anything except EAGAIN/EWOULDBLOCK/EINTR is trouble */ switch (SOCK_ERRNO) @@ -878,12 +882,6 @@ pqSendSome(PGconn *conn, int len) return -1; } } - else - { - ptr += sent; - len -= sent; - remaining -= sent; - } if (len > 0) { @@ -1048,14 +1046,11 @@ pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time) return -1; } -#ifdef USE_SSL - /* Check for SSL library buffering read bytes */ - if (forRead && conn->ssl_in_use && pgtls_read_pending(conn)) + if (forRead && io_stream_buffered_read_data(conn->io_stream)) { /* short-circuit the select */ return 1; } -#endif /* We will retry as long as we get EINTR */ do diff --git a/src/interfaces/libpq/fe-secure-gssapi.c b/src/interfaces/libpq/fe-secure-gssapi.c index ea8b0020d2..85cec6ced3 100644 --- a/src/interfaces/libpq/fe-secure-gssapi.c +++ b/src/interfaces/libpq/fe-secure-gssapi.c @@ -69,6 +69,13 @@ #define PqGSSResultNext (conn->gss_ResultNext) #define PqGSSMaxPktSize (conn->gss_MaxPktSize) +static ssize_t pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written); + +IoStreamProcessor pg_GSS_processor = { + .read = (io_stream_read_func) pg_GSS_read, + .write = (io_stream_write_func) pg_GSS_write +}; /* * Attempt to write len bytes of data from ptr to a GSSAPI-encrypted connection. @@ -82,8 +89,8 @@ * For retryable errors, caller should call again (passing the same or more * data) once the socket is ready. */ -ssize_t -pg_GSS_write(PGconn *conn, const void *ptr, size_t len) +static int +pg_GSS_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written) { OM_uint32 major, minor; @@ -94,6 +101,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) size_t bytes_encrypted; gss_ctx_id_t gctx = conn->gctx; + *bytes_written = 0; + /* * When we get a retryable failure, we must not tell the caller we have * successfully transmitted everything, else it won't retry. For @@ -124,7 +133,7 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) /* * Loop through encrypting data and sending it out until it's all done or - * pqsecure_raw_write() complains (which would likely mean that the socket + * io_stream_next_write() complains (which would likely mean that the socket * is non-blocking and the requested send() would block, or there was some * kind of actual error). */ @@ -141,20 +150,21 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) */ if (PqGSSSendLength) { - ssize_t retval; + int retval; + size_t count; ssize_t amount = PqGSSSendLength - PqGSSSendNext; - retval = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount); - if (retval <= 0) + retval = io_stream_next_write(self, PqGSSSendBuffer + PqGSSSendNext, amount, &count); + if (retval < 0 || count == 0) return retval; /* * Check if this was a partial write, and if so, move forward that * far in our buffer and try again. */ - if (retval < amount) + if (count < amount) { - PqGSSSendNext += retval; + PqGSSSendNext += count; continue; } @@ -235,7 +245,8 @@ pg_GSS_write(PGconn *conn, const void *ptr, size_t len) /* We're reporting all the data as sent, so reset PqGSSSendConsumed. */ PqGSSSendConsumed = 0; - ret = bytes_encrypted; + ret = 0; + *bytes_written = bytes_encrypted; cleanup: /* Release GSSAPI buffer storage, if we didn't already */ @@ -255,8 +266,8 @@ cleanup: * error, a message is added to conn->errorMessage. For retryable errors, * caller should call again once the socket is ready. */ -ssize_t -pg_GSS_read(PGconn *conn, void *ptr, size_t len) +static ssize_t +pg_GSS_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) { OM_uint32 major, minor; @@ -266,6 +277,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) size_t bytes_returned = 0; gss_ctx_id_t gctx = conn->gctx; + if (buffered_only) + return 0; + /* * The plan here is to read one incoming encrypted packet into * PqGSSRecvBuffer, decrypt it into PqGSSResultBuffer, and then dole out @@ -322,10 +336,10 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) /* Collect the length if we haven't already */ if (PqGSSRecvLength < sizeof(uint32)) { - ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength, - sizeof(uint32) - PqGSSRecvLength); + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + sizeof(uint32) - PqGSSRecvLength, false); - /* If ret <= 0, pqsecure_raw_read already set the correct errno */ + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -355,9 +369,9 @@ pg_GSS_read(PGconn *conn, void *ptr, size_t len) * Read as much of the packet as we are able to on this call into * wherever we left off from the last time we were called. */ - ret = pqsecure_raw_read(conn, PqGSSRecvBuffer + PqGSSRecvLength, - input.length - (PqGSSRecvLength - sizeof(uint32))); - /* If ret <= 0, pqsecure_raw_read already set the correct errno */ + ret = io_stream_next_read(self, PqGSSRecvBuffer + PqGSSRecvLength, + input.length - (PqGSSRecvLength - sizeof(uint32)), false); + /* If ret <= 0, io_stream_next_read already set the correct errno */ if (ret <= 0) return ret; @@ -418,16 +432,17 @@ cleanup: } /* - * Simple wrapper for reading from pqsecure_raw_read. + * Simple wrapper for reading from io_stream_read. Only used during connecition setup + * before GSS is added to the io_stream. * - * This takes the same arguments as pqsecure_raw_read, plus an output parameter + * This takes the same arguments as io_stream_read, plus an output parameter * to return the number of bytes read. This handles if blocking would occur and * if we detect EOF on the connection. */ static PostgresPollingStatusType gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret) { - *ret = pqsecure_raw_read(conn, recv_buffer, length); + *ret = io_stream_read(conn->io_stream, recv_buffer, length, false); if (*ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -447,7 +462,7 @@ gss_read(PGconn *conn, void *recv_buffer, size_t length, ssize_t *ret) if (!result) return PGRES_POLLING_READING; - *ret = pqsecure_raw_read(conn, recv_buffer, length); + *ret = io_stream_read(conn->io_stream, recv_buffer, length, false); if (*ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -506,8 +521,9 @@ pqsecure_open_gss(PGconn *conn) if (PqGSSSendLength) { ssize_t amount = PqGSSSendLength - PqGSSSendNext; + size_t count; - ret = pqsecure_raw_write(conn, PqGSSSendBuffer + PqGSSSendNext, amount); + ret = io_stream_write(conn->io_stream, PqGSSSendBuffer + PqGSSSendNext, amount, &count); if (ret < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) @@ -516,7 +532,7 @@ pqsecure_open_gss(PGconn *conn) return PGRES_POLLING_FAILED; } - if (ret < amount) + if (count < amount) { PqGSSSendNext += ret; return PGRES_POLLING_WRITING; @@ -662,6 +678,8 @@ pqsecure_open_gss(PGconn *conn) */ conn->gssenc = true; conn->gssapi_used = true; + io_stream_add_layer(conn->io_stream, &pg_GSS_processor, conn); + /* Clean up */ gss_release_cred(&minor, &conn->gcred); diff --git a/src/interfaces/libpq/fe-secure-openssl.c b/src/interfaces/libpq/fe-secure-openssl.c index 2b221e7d15..b9706cb151 100644 --- a/src/interfaces/libpq/fe-secure-openssl.c +++ b/src/interfaces/libpq/fe-secure-openssl.c @@ -81,8 +81,17 @@ static int PQssl_passwd_cb(char *buf, int size, int rwflag, void *userdata); static int my_sock_read(BIO *h, char *buf, int size); static int my_sock_write(BIO *h, const char *buf, int size); static BIO_METHOD *my_BIO_s_socket(void); -static int my_SSL_set_fd(PGconn *conn, int fd); - +static int my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer); +static ssize_t pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only); +static int pgtls_write(IoStreamLayer * self, PGconn *conn, void const *ptr, size_t len, size_t *bytes_written); +static bool pgtls_read_pending(PGconn *conn); +static void pgtls_close(PGconn *conn); +IoStreamProcessor pgtls_processor = { + .read = (io_stream_read_func) pgtls_read, + .write = (io_stream_write_func) pgtls_write, + .buffered_read_data = (io_stream_predicate) pgtls_read_pending, + .destroy = (io_stream_destroy_func) pgtls_close +}; static bool pq_init_ssl_lib = true; static bool pq_init_crypto_lib = true; @@ -141,8 +150,8 @@ pgtls_open_client(PGconn *conn) return open_client_SSL(conn); } -ssize_t -pgtls_read(PGconn *conn, void *ptr, size_t len) +static ssize_t +pgtls_read(IoStreamLayer * self, PGconn *conn, void *ptr, size_t len, bool buffered_only) { ssize_t n; int result_errno = 0; @@ -150,8 +159,20 @@ pgtls_read(PGconn *conn, void *ptr, size_t len) int err; unsigned long ecode; + if (buffered_only) + { + /* + * SSL_pending bytes are guaranteed to be available and readable + * without blocking + */ + len = Min(len, SSL_pending(conn->ssl)); + if (len == 0) + return 0; + } + rloop: + /* * Prepare to call SSL_get_error() by clearing thread's OpenSSL error * queue. In general, the current thread's error queue must be empty @@ -257,14 +278,14 @@ rloop: return n; } -bool +static bool pgtls_read_pending(PGconn *conn) { return SSL_pending(conn->ssl) > 0; } -ssize_t -pgtls_write(PGconn *conn, const void *ptr, size_t len) +static int +pgtls_write(IoStreamLayer * self, PGconn *conn, const void *ptr, size_t len, size_t *bytes_written) { ssize_t n; int result_errno = 0; @@ -360,7 +381,17 @@ pgtls_write(PGconn *conn, const void *ptr, size_t len) /* ensure we return the intended errno to caller */ SOCK_ERRNO_SET(result_errno); - return n; + + if (n >= 0) + { + *bytes_written = n; + return 0; + } + else + { + *bytes_written = 0; + return n; + } } char * @@ -1211,7 +1242,8 @@ initialize_SSL(PGconn *conn) */ if (!(conn->ssl = SSL_new(SSL_context)) || !SSL_set_app_data(conn->ssl, conn) || - !my_SSL_set_fd(conn, conn->sock)) + !my_SSL_set_fd(conn->ssl, conn->sock, + io_stream_add_layer(conn->io_stream, &pgtls_processor, conn))) { char *err = SSLerrmessage(ERR_get_error()); @@ -1613,7 +1645,7 @@ open_client_SSL(PGconn *conn) return PGRES_POLLING_OK; } -void +static void pgtls_close(PGconn *conn) { bool destroy_needed = false; @@ -1815,7 +1847,7 @@ PQsslAttribute(PGconn *conn, const char *attribute_name) /* * Private substitute BIO: this does the sending and receiving using - * pqsecure_raw_write() and pqsecure_raw_read() instead, to allow those + * io_stream_next_write() and io_stream_next_read() instead, to allow those * functions to disable SIGPIPE and give better error messages on I/O errors. * * These functions are closely modelled on the standard socket BIO in OpenSSL; @@ -1830,7 +1862,7 @@ my_sock_read(BIO *h, char *buf, int size) { int res; - res = pqsecure_raw_read((PGconn *) BIO_get_app_data(h), buf, size); + res = io_stream_next_read(BIO_get_app_data(h), buf, size, false); BIO_clear_retry_flags(h); if (res < 0) { @@ -1859,8 +1891,9 @@ static int my_sock_write(BIO *h, const char *buf, int size) { int res; + size_t count; - res = pqsecure_raw_write((PGconn *) BIO_get_app_data(h), buf, size); + res = io_stream_next_write(BIO_get_app_data(h), buf, size, &count); BIO_clear_retry_flags(h); if (res < 0) { @@ -1882,7 +1915,7 @@ my_sock_write(BIO *h, const char *buf, int size) } } - return res; + return res == 0 ? count : res; } static BIO_METHOD * @@ -1952,7 +1985,7 @@ err: /* This should exactly match OpenSSL's SSL_set_fd except for using my BIO */ static int -my_SSL_set_fd(PGconn *conn, int fd) +my_SSL_set_fd(SSL *ssl, int fd, IoStreamLayer * layer) { int ret = 0; BIO *bio; @@ -1965,15 +1998,16 @@ my_SSL_set_fd(PGconn *conn, int fd) goto err; } bio = BIO_new(bio_method); + if (bio == NULL) { SSLerr(SSL_F_SSL_SET_FD, ERR_R_BUF_LIB); goto err; } - BIO_set_app_data(bio, conn); + BIO_set_app_data(bio, layer); - SSL_set_bio(conn->ssl, bio, bio); BIO_set_fd(bio, fd, BIO_NOCLOSE); + SSL_set_bio(ssl, bio, bio); ret = 1; err: return ret; diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index b2430362a9..4b34de0220 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -45,55 +45,6 @@ #include "libpq-fe.h" #include "libpq-int.h" -/* - * Macros to handle disabling and then restoring the state of SIGPIPE handling. - * On Windows, these are all no-ops since there's no SIGPIPEs. - */ - -#ifndef WIN32 - -#define SIGPIPE_MASKED(conn) ((conn)->sigpipe_so || (conn)->sigpipe_flag) - -struct sigpipe_info -{ - sigset_t oldsigmask; - bool sigpipe_pending; - bool got_epipe; -}; - -#define DECLARE_SIGPIPE_INFO(spinfo) struct sigpipe_info spinfo - -#define DISABLE_SIGPIPE(conn, spinfo, failaction) \ - do { \ - (spinfo).got_epipe = false; \ - if (!SIGPIPE_MASKED(conn)) \ - { \ - if (pq_block_sigpipe(&(spinfo).oldsigmask, \ - &(spinfo).sigpipe_pending) < 0) \ - failaction; \ - } \ - } while (0) - -#define REMEMBER_EPIPE(spinfo, cond) \ - do { \ - if (cond) \ - (spinfo).got_epipe = true; \ - } while (0) - -#define RESTORE_SIGPIPE(conn, spinfo) \ - do { \ - if (!SIGPIPE_MASKED(conn)) \ - pq_reset_sigpipe(&(spinfo).oldsigmask, (spinfo).sigpipe_pending, \ - (spinfo).got_epipe); \ - } while (0) -#else /* WIN32 */ - -#define DECLARE_SIGPIPE_INFO(spinfo) -#define DISABLE_SIGPIPE(conn, spinfo, failaction) -#define REMEMBER_EPIPE(spinfo, cond) -#define RESTORE_SIGPIPE(conn, spinfo) -#endif /* WIN32 */ - /* ------------------------------------------------------------ */ /* Procedures common to all secure sessions */ /* ------------------------------------------------------------ */ @@ -160,281 +111,6 @@ pqsecure_open_client(PGconn *conn) #endif } -/* - * Close secure session. - */ -void -pqsecure_close(PGconn *conn) -{ -#ifdef USE_SSL - pgtls_close(conn); -#endif -} - -/* - * Read data from a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -ssize_t -pqsecure_read(PGconn *conn, void *ptr, size_t len) -{ - ssize_t n; - -#ifdef USE_SSL - if (conn->ssl_in_use) - { - n = pgtls_read(conn, ptr, len); - } - else -#endif -#ifdef ENABLE_GSS - if (conn->gssenc) - { - n = pg_GSS_read(conn, ptr, len); - } - else -#endif - { - n = pqsecure_raw_read(conn, ptr, len); - } - - return n; -} - -ssize_t -pqsecure_raw_read(PGconn *conn, void *ptr, size_t len) -{ - ssize_t n; - int result_errno = 0; - char sebuf[PG_STRERROR_R_BUFLEN]; - - SOCK_ERRNO_SET(0); - - n = recv(conn->sock, ptr, len, 0); - - if (n < 0) - { - result_errno = SOCK_ERRNO; - - /* Set error message if appropriate */ - switch (result_errno) - { -#ifdef EAGAIN - case EAGAIN: -#endif -#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - case EWOULDBLOCK: -#endif - case EINTR: - /* no error message, caller is expected to retry */ - break; - - case EPIPE: - case ECONNRESET: - libpq_append_conn_error(conn, "server closed the connection unexpectedly\n" - "\tThis probably means the server terminated abnormally\n" - "\tbefore or while processing the request."); - break; - - case 0: - /* If errno didn't get set, treat it as regular EOF */ - n = 0; - break; - - default: - libpq_append_conn_error(conn, "could not receive data from server: %s", - SOCK_STRERROR(result_errno, - sebuf, sizeof(sebuf))); - break; - } - } - - /* ensure we return the intended errno to caller */ - SOCK_ERRNO_SET(result_errno); - - return n; -} - -/* - * Write data to a secure connection. - * - * Returns the number of bytes written, or a negative value (with errno - * set) upon failure. The write count could be less than requested. - * - * Note that socket-level hard failures are masked from the caller, - * instead setting conn->write_failed and storing an error message - * in conn->write_err_msg; see pqsecure_raw_write. This allows us to - * postpone reporting of write failures until we're sure no error - * message is available from the server. - * - * However, errors detected in the SSL or GSS management level are reported - * via a negative result, with message appended to conn->errorMessage. - * It's frequently unclear whether such errors should be considered read or - * write errors, so we don't attempt to postpone reporting them. - * - * The caller must still inspect errno upon failure, but only to determine - * whether to continue/retry; a message has been saved someplace in any case. - */ -ssize_t -pqsecure_write(PGconn *conn, const void *ptr, size_t len) -{ - ssize_t n; - -#ifdef USE_SSL - if (conn->ssl_in_use) - { - n = pgtls_write(conn, ptr, len); - } - else -#endif -#ifdef ENABLE_GSS - if (conn->gssenc) - { - n = pg_GSS_write(conn, ptr, len); - } - else -#endif - { - n = pqsecure_raw_write(conn, ptr, len); - } - - return n; -} - -/* - * Low-level implementation of pqsecure_write. - * - * This is used directly for an unencrypted connection. For encrypted - * connections, this does the physical I/O on behalf of pgtls_write or - * pg_GSS_write. - * - * This function reports failure (i.e., returns a negative result) only - * for retryable errors such as EINTR. Looping for such cases is to be - * handled at some outer level, maybe all the way up to the application. - * For hard failures, we set conn->write_failed and store an error message - * in conn->write_err_msg, but then claim to have written the data anyway. - * This is because we don't want to report write failures so long as there - * is a possibility of reading from the server and getting an error message - * that could explain why the connection dropped. Many TCP stacks have - * race conditions such that a write failure may or may not be reported - * before all incoming data has been read. - * - * Note that this error behavior happens below the SSL management level when - * we are using SSL. That's because at least some versions of OpenSSL are - * too quick to report a write failure when there's still a possibility to - * get a more useful error from the server. - */ -ssize_t -pqsecure_raw_write(PGconn *conn, const void *ptr, size_t len) -{ - ssize_t n; - int flags = 0; - int result_errno = 0; - char msgbuf[1024]; - char sebuf[PG_STRERROR_R_BUFLEN]; - - DECLARE_SIGPIPE_INFO(spinfo); - - /* - * If we already had a write failure, we will never again try to send data - * on that connection. Even if the kernel would let us, we've probably - * lost message boundary sync with the server. conn->write_failed - * therefore persists until the connection is reset, and we just discard - * all data presented to be written. - */ - if (conn->write_failed) - return len; - -#ifdef MSG_NOSIGNAL - if (conn->sigpipe_flag) - flags |= MSG_NOSIGNAL; - -retry_masked: -#endif /* MSG_NOSIGNAL */ - - DISABLE_SIGPIPE(conn, spinfo, return -1); - - n = send(conn->sock, ptr, len, flags); - - if (n < 0) - { - result_errno = SOCK_ERRNO; - - /* - * If we see an EINVAL, it may be because MSG_NOSIGNAL isn't available - * on this machine. So, clear sigpipe_flag so we don't try the flag - * again, and retry the send(). - */ -#ifdef MSG_NOSIGNAL - if (flags != 0 && result_errno == EINVAL) - { - conn->sigpipe_flag = false; - flags = 0; - goto retry_masked; - } -#endif /* MSG_NOSIGNAL */ - - /* Set error message if appropriate */ - switch (result_errno) - { -#ifdef EAGAIN - case EAGAIN: -#endif -#if defined(EWOULDBLOCK) && (!defined(EAGAIN) || (EWOULDBLOCK != EAGAIN)) - case EWOULDBLOCK: -#endif - case EINTR: - /* no error message, caller is expected to retry */ - break; - - case EPIPE: - /* Set flag for EPIPE */ - REMEMBER_EPIPE(spinfo, true); - - /* FALL THRU */ - - case ECONNRESET: - conn->write_failed = true; - /* Store error message in conn->write_err_msg, if possible */ - /* (strdup failure is OK, we'll cope later) */ - snprintf(msgbuf, sizeof(msgbuf), - libpq_gettext("server closed the connection unexpectedly\n" - "\tThis probably means the server terminated abnormally\n" - "\tbefore or while processing the request.")); - /* keep newline out of translated string */ - strlcat(msgbuf, "\n", sizeof(msgbuf)); - conn->write_err_msg = strdup(msgbuf); - /* Now claim the write succeeded */ - n = len; - break; - - default: - conn->write_failed = true; - /* Store error message in conn->write_err_msg, if possible */ - /* (strdup failure is OK, we'll cope later) */ - snprintf(msgbuf, sizeof(msgbuf), - libpq_gettext("could not send data to server: %s"), - SOCK_STRERROR(result_errno, - sebuf, sizeof(sebuf))); - /* keep newline out of translated string */ - strlcat(msgbuf, "\n", sizeof(msgbuf)); - conn->write_err_msg = strdup(msgbuf); - /* Now claim the write succeeded */ - n = len; - break; - } - } - - RESTORE_SIGPIPE(conn, spinfo); - - /* ensure we return the intended errno to caller */ - SOCK_ERRNO_SET(result_errno); - - return n; -} /* Dummy versions of SSL info functions, when built without SSL support */ #ifndef USE_SSL @@ -507,90 +183,3 @@ PQgssEncInUse(PGconn *conn) } #endif /* ENABLE_GSS */ - - -#if !defined(WIN32) - -/* - * Block SIGPIPE for this thread. This prevents send()/write() from exiting - * the application. - */ -int -pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending) -{ - sigset_t sigpipe_sigset; - sigset_t sigset; - - sigemptyset(&sigpipe_sigset); - sigaddset(&sigpipe_sigset, SIGPIPE); - - /* Block SIGPIPE and save previous mask for later reset */ - SOCK_ERRNO_SET(pthread_sigmask(SIG_BLOCK, &sigpipe_sigset, osigset)); - if (SOCK_ERRNO) - return -1; - - /* We can have a pending SIGPIPE only if it was blocked before */ - if (sigismember(osigset, SIGPIPE)) - { - /* Is there a pending SIGPIPE? */ - if (sigpending(&sigset) != 0) - return -1; - - if (sigismember(&sigset, SIGPIPE)) - *sigpipe_pending = true; - else - *sigpipe_pending = false; - } - else - *sigpipe_pending = false; - - return 0; -} - -/* - * Discard any pending SIGPIPE and reset the signal mask. - * - * Note: we are effectively assuming here that the C library doesn't queue - * up multiple SIGPIPE events. If it did, then we'd accidentally leave - * ours in the queue when an event was already pending and we got another. - * As long as it doesn't queue multiple events, we're OK because the caller - * can't tell the difference. - * - * The caller should say got_epipe = false if it is certain that it - * didn't get an EPIPE error; in that case we'll skip the clear operation - * and things are definitely OK, queuing or no. If it got one or might have - * gotten one, pass got_epipe = true. - * - * We do not want this to change errno, since if it did that could lose - * the error code from a preceding send(). We essentially assume that if - * we were able to do pq_block_sigpipe(), this can't fail. - */ -void -pq_reset_sigpipe(sigset_t *osigset, bool sigpipe_pending, bool got_epipe) -{ - int save_errno = SOCK_ERRNO; - int signo; - sigset_t sigset; - - /* Clear SIGPIPE only if none was pending */ - if (got_epipe && !sigpipe_pending) - { - if (sigpending(&sigset) == 0 && - sigismember(&sigset, SIGPIPE)) - { - sigset_t sigpipe_sigset; - - sigemptyset(&sigpipe_sigset); - sigaddset(&sigpipe_sigset, SIGPIPE); - - sigwait(&sigpipe_sigset, &signo); - } - } - - /* Restore saved block mask */ - pthread_sigmask(SIG_SETMASK, osigset, NULL); - - SOCK_ERRNO_SET(save_errno); -} - -#endif /* !WIN32 */ diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 7888199b0d..1314663213 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -81,6 +81,7 @@ typedef struct #endif #endif /* USE_OPENSSL */ +#include "common/io_stream.h" #include "common/pg_prng.h" /* @@ -456,6 +457,7 @@ struct pg_conn PGcmdQueueEntry *cmd_queue_recycle; /* Connection data */ + IoStream *io_stream; pgsocket sock; /* FD for socket, PGINVALID_SOCKET if * unconnected */ SockAddr laddr; /* Local address */ @@ -753,11 +755,6 @@ extern int pqWriteReady(PGconn *conn); extern int pqsecure_initialize(PGconn *, bool, bool); extern PostgresPollingStatusType pqsecure_open_client(PGconn *); -extern void pqsecure_close(PGconn *); -extern ssize_t pqsecure_read(PGconn *, void *ptr, size_t len); -extern ssize_t pqsecure_write(PGconn *, const void *ptr, size_t len); -extern ssize_t pqsecure_raw_read(PGconn *, void *ptr, size_t len); -extern ssize_t pqsecure_raw_write(PGconn *, const void *ptr, size_t len); #if !defined(WIN32) extern int pq_block_sigpipe(sigset_t *osigset, bool *sigpipe_pending); @@ -793,34 +790,6 @@ extern int pgtls_init(PGconn *conn, bool do_ssl, bool do_crypto); */ extern PostgresPollingStatusType pgtls_open_client(PGconn *conn); -/* - * Close SSL connection. - */ -extern void pgtls_close(PGconn *conn); - -/* - * Read data from a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -extern ssize_t pgtls_read(PGconn *conn, void *ptr, size_t len); - -/* - * Is there unread data waiting in the SSL read buffer? - */ -extern bool pgtls_read_pending(PGconn *conn); - -/* - * Write data to a secure connection. - * - * On failure, this function is responsible for appending a suitable message - * to conn->errorMessage. The caller must still inspect errno, but only - * to determine whether to continue/retry after error. - */ -extern ssize_t pgtls_write(PGconn *conn, const void *ptr, size_t len); - /* * Get the hash of the server certificate, for SCRAM channel binding type * tls-server-end-point. @@ -851,13 +820,6 @@ extern int pgtls_verify_peer_name_matches_certificate_guts(PGconn *conn, * Establish a GSSAPI-encrypted connection. */ extern PostgresPollingStatusType pqsecure_open_gss(PGconn *conn); - -/* - * Read and write functions for GSSAPI-encrypted connections, with internal - * buffering to handle nonblocking sockets. - */ -extern ssize_t pg_GSS_write(PGconn *conn, const void *ptr, size_t len); -extern ssize_t pg_GSS_read(PGconn *conn, void *ptr, size_t len); #endif /* === in fe-trace.c === */ -- 2.42.0 ^ permalink raw reply [nested|flat] 80+ messages in thread
* Re: libpq compression (part 3) @ 2023-12-20 20:49 Robert Haas <[email protected]> parent: Jacob Burroughs <[email protected]> 1 sibling, 0 replies; 80+ messages in thread From: Robert Haas @ 2023-12-20 20:49 UTC (permalink / raw) To: Jacob Burroughs <[email protected]>; +Cc: [email protected] On Tue, Dec 19, 2023 at 11:41 AM Jacob Burroughs <[email protected]> wrote: > The compression "handshaking" process now looks as follows: > 1. Client sends startup packet with `_pq_.libpq_compression = alg1;alg2` > 2. At this point, the server can immediately begin compressing packets > to the client with any of the specified algorithms it supports if it > so chooses > 3. Server includes `libpq_compression` in the automatically sent > `ParameterStatus` messages before handshaking > 4. At this point, the client can immediately begin compressing packets > to the server with any of the supported algorithms > Both the server and client will prefer to compress using the first > algorithm in their list that the other side supports, and we > explicitly support `none` in the algorithm list. This allows e.g. a > client to use `none;gzip` and a server to use `zstd;gzip;lz4`, and > then the client will not compress its data but the server will send > its data using gzip. I'm having difficulty understanding the details of this handshaking algorithm from this description. It seems good that the handshake proceeds in each direction somewhat separately from the other, but I don't quite understand how the whole thing fits together. If the client tells the server that 'none,gzip' is supported, and I elect to start using gzip, how does the client know that I picked gzip rather than none? Are the compressed packets self-identifying? It's also slightly odd to me that the same parameter seems to specify both what we want to send, and what we're able to receive. I'm not really sure we should have separate parameters for those things, but I don't quite understand how this works without it. The "none" thing seems like a bit of a hack. It lets you say "I'd like to receive compressed data but send uncompressed data" ... but what about the reverse? How do you say "don't bother compressing what you receive from the server, but please lz4 everything you send to the server"? Or how do you say "use zstd from server to client, but lz4 from client to server"? It seems like you can't really say that kind of thing. What if we had, on the server side, a GUC saying what compression to accept and a GUC saying what compression to be willing to do? And then let the client request whatever it wants for each direction. > Also please let me know if I have made any notable mailing list/patch > etiquette/format/structure errors. This is my first time submitting a > patch to a mailing-list driven open source project and while I have > tried to carefully review the various wiki guides I'm sure I didn't > take everything in perfectly. Seems fine to me. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 80+ messages in thread
end of thread, other threads:[~2023-12-20 20:49 UTC | newest] Thread overview: 80+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v2 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v7 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v6 1/2] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v11 1/9] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH 1/5] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v1] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v12 1/5] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v8 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v4 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v10 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v3 1/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v5 2/3] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]> 2023-12-19 16:40 libpq compression (part 3) Jacob Burroughs <[email protected]> 2023-12-19 17:02 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]> 2023-12-20 19:39 ` Re: libpq compression (part 3) Jacob Burroughs <[email protected]> 2023-12-20 20:49 ` Re: libpq compression (part 3) Robert Haas <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox