public inbox for [email protected]
help / color / mirror / Atom feed[PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table
19+ messages / 5 participants
[nested] [flat]
* [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table
@ 2020-06-06 22:42 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw)
---
doc/src/sgml/ddl.sgml | 4 +-
doc/src/sgml/ref/create_index.sgml | 9 --
src/backend/commands/indexcmds.c | 172 +++++++++++++++++--------
src/test/regress/expected/indexing.out | 127 +++++++++++++++++-
src/test/regress/sql/indexing.sql | 26 +++-
5 files changed, 268 insertions(+), 70 deletions(-)
diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml
index 38618de01c5..cd72b455447 100644
--- a/doc/src/sgml/ddl.sgml
+++ b/doc/src/sgml/ddl.sgml
@@ -4163,9 +4163,7 @@ ALTER TABLE measurement ATTACH PARTITION measurement_y2008m02
so that they are applied automatically to the entire hierarchy.
This is very
convenient, as not only will the existing partitions become indexed, but
- also any partitions that are created in the future will. One limitation is
- that it's not possible to use the <literal>CONCURRENTLY</literal>
- qualifier when creating such a partitioned index. To avoid long lock
+ also any partitions that are created in the future will. To avoid long lock
times, it is possible to use <command>CREATE INDEX ON ONLY</command>
the partitioned table; such an index is marked invalid, and the partitions
do not get the index applied automatically. The indexes on partitions can
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index 40986aa502f..fc8cda655f0 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -692,15 +692,6 @@ Indexes:
cannot.
</para>
- <para>
- Concurrent builds for indexes on partitioned tables are currently not
- supported. However, you may concurrently build the index on each
- partition individually and then finally create the partitioned index
- non-concurrently in order to reduce the time where writes to the
- partitioned table will be locked out. In this case, building the
- partitioned index is a metadata only operation.
- </para>
-
</refsect2>
</refsect1>
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index b5b860c3abf..cfab45b9992 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -92,6 +92,11 @@ static char *ChooseIndexName(const char *tabname, Oid namespaceId,
bool primary, bool isconstraint);
static char *ChooseIndexNameAddition(List *colnames);
static List *ChooseIndexColumnNames(List *indexElems);
+static void DefineIndexConcurrentInternal(Oid relationId,
+ Oid indexRelationId,
+ IndexInfo *indexInfo,
+ LOCKTAG heaplocktag,
+ LockRelId heaprelid);
static void ReindexIndex(RangeVar *indexRelation, ReindexParams *params,
bool isTopLevel);
static void RangeVarCallbackForReindexIndex(const RangeVar *relation,
@@ -551,7 +556,6 @@ DefineIndex(Oid relationId,
bool amcanorder;
amoptions_function amoptions;
bool partitioned;
- bool safe_index;
Datum reloptions;
int16 *coloptions;
IndexInfo *indexInfo;
@@ -559,12 +563,10 @@ DefineIndex(Oid relationId,
bits16 constr_flags;
int numberOfAttributes;
int numberOfKeyAttributes;
- TransactionId limitXmin;
ObjectAddress address;
LockRelId heaprelid;
LOCKTAG heaplocktag;
LOCKMODE lockmode;
- Snapshot snapshot;
Oid root_save_userid;
int root_save_sec_context;
int root_save_nestlevel;
@@ -697,17 +699,6 @@ DefineIndex(Oid relationId,
partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE;
if (partitioned)
{
- /*
- * Note: we check 'stmt->concurrent' rather than 'concurrent', so that
- * the error is thrown also for temporary tables. Seems better to be
- * consistent, even though we could do it on temporary table because
- * we're not actually doing it concurrently.
- */
- if (stmt->concurrent)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot create index on partitioned table \"%s\" concurrently",
- RelationGetRelationName(rel))));
if (stmt->excludeOpNames)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1079,10 +1070,6 @@ DefineIndex(Oid relationId,
}
}
- /* Is index safe for others to ignore? See set_indexsafe_procflags() */
- safe_index = indexInfo->ii_Expressions == NIL &&
- indexInfo->ii_Predicate == NIL;
-
/*
* Report index creation if appropriate (delay this till after most of the
* error checks)
@@ -1147,6 +1134,11 @@ DefineIndex(Oid relationId,
if (pd->nparts != 0)
flags |= INDEX_CREATE_INVALID;
}
+ else if (concurrent && OidIsValid(parentIndexId))
+ {
+ /* If concurrent, initially build index partitions as "invalid" */
+ flags |= INDEX_CREATE_INVALID;
+ }
if (stmt->deferrable)
constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE;
@@ -1427,12 +1419,15 @@ DefineIndex(Oid relationId,
createdConstraintId,
is_alter_table, check_rights, check_not_in_use,
skip_build, quiet);
+
SetUserIdAndSecContext(child_save_userid,
child_save_sec_context);
}
- pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- i + 1);
+ /* For concurrent build, this is a catalog-only stage */
+ if (!concurrent)
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ i + 1);
free_attrmap(attmap);
}
@@ -1444,46 +1439,39 @@ DefineIndex(Oid relationId,
* invalid, this is incorrect, so update our row to invalid too.
*/
if (invalidate_parent)
- {
- Relation pg_index = table_open(IndexRelationId, RowExclusiveLock);
- HeapTuple tup,
- newtup;
-
- tup = SearchSysCache1(INDEXRELID,
- ObjectIdGetDatum(indexRelationId));
- if (!HeapTupleIsValid(tup))
- elog(ERROR, "cache lookup failed for index %u",
- indexRelationId);
- newtup = heap_copytuple(tup);
- ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false;
- CatalogTupleUpdate(pg_index, &tup->t_self, newtup);
- ReleaseSysCache(tup);
- table_close(pg_index, RowExclusiveLock);
- heap_freetuple(newtup);
- }
+ index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID);
}
/*
* Indexes on partitioned tables are not themselves built, so we're
- * done here.
+ * done here in the non-concurrent case.
*/
- AtEOXact_GUC(false, root_save_nestlevel);
- SetUserIdAndSecContext(root_save_userid, root_save_sec_context);
- table_close(rel, NoLock);
- if (!OidIsValid(parentIndexId))
- pgstat_progress_end_command();
- return address;
+
+ if (!concurrent)
+ {
+ AtEOXact_GUC(false, root_save_nestlevel);
+ SetUserIdAndSecContext(root_save_userid, root_save_sec_context);
+ table_close(rel, NoLock);
+
+ if (!OidIsValid(parentIndexId))
+ pgstat_progress_end_command();
+
+ return address;
+ }
}
AtEOXact_GUC(false, root_save_nestlevel);
SetUserIdAndSecContext(root_save_userid, root_save_sec_context);
- if (!concurrent)
+ /*
+ * All done in the non-concurrent case, and when building catalog entries
+ * of partitions for CIC.
+ */
+ if (!concurrent || OidIsValid(parentIndexId))
{
- /* Close the heap and we're done, in the non-concurrent case */
table_close(rel, NoLock);
- /* If this is the top-level index, we're done. */
+ /* If this is the top-level index, the command is complete. */
if (!OidIsValid(parentIndexId))
pgstat_progress_end_command();
@@ -1495,6 +1483,92 @@ DefineIndex(Oid relationId,
SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId);
table_close(rel, NoLock);
+ if (!partitioned)
+ {
+ DefineIndexConcurrentInternal(relationId, indexRelationId,
+ indexInfo, heaplocktag, heaprelid);
+ pgstat_progress_end_command();
+ return address;
+ }
+ else
+ {
+ /* finish CIC by building indexes on partitions */
+ ListCell *lc;
+ List *childs;
+ int npart = 0;
+ MemoryContext cic_context,
+ old_context;
+
+ /*
+ * Create special memory context for cross-transaction storage.
+ */
+ cic_context = AllocSetContextCreate(PortalContext,
+ "Create index concurrently",
+ ALLOCSET_DEFAULT_SIZES);
+
+ old_context = MemoryContextSwitchTo(cic_context);
+ childs = find_all_inheritors(indexRelationId, ShareLock, NULL);
+ MemoryContextSwitchTo(old_context);
+
+ foreach(lc, childs)
+ {
+ Oid indrelid = lfirst_oid(lc);
+ Oid tabrelid = IndexGetRelation(indrelid, false);
+
+ if (RELKIND_HAS_STORAGE(get_rel_relkind(indrelid)) &&
+ !get_index_isvalid(indrelid))
+ {
+ rel = table_open(relationId, ShareUpdateExclusiveLock);
+ heaprelid = rel->rd_lockInfo.lockRelId;
+ table_close(rel, ShareUpdateExclusiveLock);
+ SET_LOCKTAG_RELATION(heaplocktag, heaprelid.dbId, heaprelid.relId);
+
+ /* Process each partition in a separate transaction */
+ DefineIndexConcurrentInternal(tabrelid, indrelid, indexInfo,
+ heaplocktag, heaprelid);
+
+ PushActiveSnapshot(GetTransactionSnapshot());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ ++npart);
+ }
+
+ /* Set all indexes as valid, including the parent */
+ foreach(lc, childs)
+ {
+ Oid indrelid = lfirst_oid(lc);
+
+ if (get_rel_relkind(indrelid) != RELKIND_PARTITIONED_INDEX)
+ continue;
+ if (get_index_isvalid(indrelid))
+ continue;
+
+ index_set_state_flags(indrelid, INDEX_CREATE_SET_READY);
+ CommandCounterIncrement();
+ index_set_state_flags(indrelid, INDEX_CREATE_SET_VALID);
+ }
+
+ MemoryContextDelete(cic_context);
+ pgstat_progress_end_command();
+ PopActiveSnapshot();
+ return address;
+ }
+}
+
+
+static void
+DefineIndexConcurrentInternal(Oid relationId,
+ Oid indexRelationId, IndexInfo *indexInfo,
+ LOCKTAG heaplocktag, LockRelId heaprelid)
+{
+ TransactionId limitXmin;
+ Snapshot snapshot;
+
+ /* Is index safe for others to ignore? See set_indexsafe_procflags() */
+ bool safe_index = indexInfo->ii_Expressions == NIL &&
+ indexInfo->ii_Predicate == NIL;
+
/*
* For a concurrent build, it's important to make the catalog entries
* visible to other transactions before we start to build the index. That
@@ -1690,10 +1764,6 @@ DefineIndex(Oid relationId,
* Last thing to do is release the session-level lock on the parent table.
*/
UnlockRelationIdForSession(&heaprelid, ShareUpdateExclusiveLock);
-
- pgstat_progress_end_command();
-
- return address;
}
diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out
index 1bdd430f063..f1beee6d240 100644
--- a/src/test/regress/expected/indexing.out
+++ b/src/test/regress/expected/indexing.out
@@ -50,11 +50,130 @@ select relname, relkind, relhassubclass, inhparent::regclass
(8 rows)
drop table idxpart;
--- Some unsupported features
+-- CIC on partitioned table
create table idxpart (a int, b int, c text) partition by range (a);
-create table idxpart1 partition of idxpart for values from (0) to (10);
-create index concurrently on idxpart (a);
-ERROR: cannot create index on partitioned table "idxpart" concurrently
+create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a);
+create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a);
+create table idxpart111 partition of idxpart11 default partition by range(a);
+create table idxpart1111 partition of idxpart111 default partition by range(a);
+create table idxpart2 partition of idxpart for values from (10) to (20);
+create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a);
+create table idxpart31 partition of idxpart3 default;
+insert into idxpart2 values(10),(10); -- not unique
+create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves
+create index concurrently on idxpart1 (a); -- partitioned and partition
+create index concurrently on idxpart2 (a); -- leaf
+create index concurrently on idxpart (a); -- partitioned
+create unique index concurrently on idxpart (a); -- partitioned, unique failure
+ERROR: could not create unique index "idxpart2_a_idx1"
+DETAIL: Key (a)=(10) is duplicated.
+\d idxpart
+ Partitioned table "public.idxpart"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition key: RANGE (a)
+Indexes:
+ "idxpart_a_idx" btree (a)
+ "idxpart_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 3 (Use \d+ to list them.)
+
+\d idxpart1
+ Partitioned table "public.idxpart1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart FOR VALUES FROM (0) TO (10)
+Partition key: RANGE (a)
+Indexes:
+ "idxpart1_a_idx" btree (a)
+ "idxpart1_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 1 (Use \d+ to list them.)
+
+\d idxpart11
+ Partitioned table "public.idxpart11"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart1 FOR VALUES FROM (0) TO (10)
+Partition key: RANGE (a)
+Indexes:
+ "idxpart11_a_idx" btree (a)
+ "idxpart11_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 1 (Use \d+ to list them.)
+
+\d idxpart111
+ Partitioned table "public.idxpart111"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart11 DEFAULT
+Partition key: RANGE (a)
+Indexes:
+ "idxpart111_a_idx" btree (a)
+ "idxpart111_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 1 (Use \d+ to list them.)
+
+\d idxpart1111
+ Partitioned table "public.idxpart1111"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart111 DEFAULT
+Partition key: RANGE (a)
+Indexes:
+ "idxpart1111_a_idx" btree (a)
+ "idxpart1111_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 0
+
+\d idxpart2
+ Table "public.idxpart2"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart FOR VALUES FROM (10) TO (20)
+Indexes:
+ "idxpart2_a_idx" btree (a)
+ "idxpart2_a_idx1" UNIQUE, btree (a) INVALID
+
+\d idxpart3
+ Partitioned table "public.idxpart3"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart FOR VALUES FROM (30) TO (40)
+Partition key: RANGE (a)
+Indexes:
+ "idxpart3_a_idx" btree (a)
+ "idxpart3_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 1 (Use \d+ to list them.)
+
+\d idxpart31
+ Table "public.idxpart31"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart3 DEFAULT
+Indexes:
+ "idxpart31_a_idx" btree (a)
+ "idxpart31_a_idx1" UNIQUE, btree (a) INVALID
+
drop table idxpart;
-- Verify bugfix with query on indexed partitioned table with no partitions
-- https://postgr.es/m/[email protected]
diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql
index 429120e7104..fb0baedcc28 100644
--- a/src/test/regress/sql/indexing.sql
+++ b/src/test/regress/sql/indexing.sql
@@ -29,10 +29,30 @@ select relname, relkind, relhassubclass, inhparent::regclass
where relname like 'idxpart%' order by relname;
drop table idxpart;
--- Some unsupported features
+-- CIC on partitioned table
create table idxpart (a int, b int, c text) partition by range (a);
-create table idxpart1 partition of idxpart for values from (0) to (10);
-create index concurrently on idxpart (a);
+create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a);
+create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a);
+create table idxpart111 partition of idxpart11 default partition by range(a);
+create table idxpart1111 partition of idxpart111 default partition by range(a);
+create table idxpart2 partition of idxpart for values from (10) to (20);
+create table idxpart3 partition of idxpart for values from (30) to (40) partition by range(a);
+create table idxpart31 partition of idxpart3 default;
+
+insert into idxpart2 values(10),(10); -- not unique
+create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves
+create index concurrently on idxpart1 (a); -- partitioned and partition
+create index concurrently on idxpart2 (a); -- leaf
+create index concurrently on idxpart (a); -- partitioned
+create unique index concurrently on idxpart (a); -- partitioned, unique failure
+\d idxpart
+\d idxpart1
+\d idxpart11
+\d idxpart111
+\d idxpart1111
+\d idxpart2
+\d idxpart3
+\d idxpart31
drop table idxpart;
-- Verify bugfix with query on indexed partitioned table with no partitions
--
2.25.1
--zCKi3GIZzVBPywwA--
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
@ 2023-03-07 02:47 Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Julien Rouhaud @ 2023-03-07 02:47 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Gregory Stark (as CFM) <[email protected]>; Pavel Stehule <[email protected]>; Justin Pryzby <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi,
On Mon, Mar 06, 2023 at 10:20:32PM +0100, Daniel Gustafsson wrote:
> > On 6 Mar 2023, at 21:45, Gregory Stark (as CFM) <[email protected]> wrote:
> >
> > So.... This patch has been through a lot of commitfests. And it really
> > doesn't seem that hard to resolve -- Pavel has seemingly been willing
> > to go along whichever way the wind has been blowing but honestly it
> > kind of seems like he's just gotten drive-by suggestions and he's put
> > a lot of work into trying to satisfy them.
>
> Agreed.
Indeed, I'm not sure I would have had that much patience.
> > He implemented --include-tables-from-file=... etc. Then he implemented
> > a hand-written parser for a DSL to select objects, then he implemented
> > a bison parser, then he went back to the hand-written parser.
>
> Well, kind of. I was trying to take the patch to the finishing line but was
> uncomfortable with the hand written parser so I implemented a parser in Bison
> to replace it with. Not that hand-written parsers are bad per se (or that my
> bison parser was perfect), but reading quoted identifiers across line
> boundaries tend to require a fair amount of handwritten code. Pavel did not
> object to this version, but it was objected to by two other committers.
>
> At this point [0] I stepped down from trying to finish it as the approach I was
> comfortable didn't gain traction (which is totally fine).
>
> Downthread from this the patch got a lot of reviews from Julien with the old
> parser back in place.
Yeah, and the current state seems quite good to me.
> > Can we get some consensus on whether the DSL looks right
>
> I would consider this pretty settled.
Agreed.
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
@ 2023-03-16 12:05 ` Pavel Stehule <[email protected]>
2023-03-18 19:48 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
0 siblings, 2 replies; 19+ messages in thread
From: Pavel Stehule @ 2023-03-16 12:05 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Justin Pryzby <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
út 7. 3. 2023 v 3:47 odesílatel Julien Rouhaud <[email protected]> napsal:
> Hi,
>
> On Mon, Mar 06, 2023 at 10:20:32PM +0100, Daniel Gustafsson wrote:
> > > On 6 Mar 2023, at 21:45, Gregory Stark (as CFM) <[email protected]>
> wrote:
> > >
> > > So.... This patch has been through a lot of commitfests. And it really
> > > doesn't seem that hard to resolve -- Pavel has seemingly been willing
> > > to go along whichever way the wind has been blowing but honestly it
> > > kind of seems like he's just gotten drive-by suggestions and he's put
> > > a lot of work into trying to satisfy them.
> >
> > Agreed.
>
> Indeed, I'm not sure I would have had that much patience.
>
> > > He implemented --include-tables-from-file=... etc. Then he implemented
> > > a hand-written parser for a DSL to select objects, then he implemented
> > > a bison parser, then he went back to the hand-written parser.
> >
> > Well, kind of. I was trying to take the patch to the finishing line but
> was
> > uncomfortable with the hand written parser so I implemented a parser in
> Bison
> > to replace it with. Not that hand-written parsers are bad per se (or
> that my
> > bison parser was perfect), but reading quoted identifiers across line
> > boundaries tend to require a fair amount of handwritten code. Pavel did
> not
> > object to this version, but it was objected to by two other committers.
> >
> > At this point [0] I stepped down from trying to finish it as the
> approach I was
> > comfortable didn't gain traction (which is totally fine).
> >
> > Downthread from this the patch got a lot of reviews from Julien with the
> old
> > parser back in place.
>
> Yeah, and the current state seems quite good to me.
>
> > > Can we get some consensus on whether the DSL looks right
> >
> > I would consider this pretty settled.
>
> Agreed.
>
rebase + enhancing about related option from a563c24
Regards
Pavel
Attachments:
[text/x-patch] 0001-possibility-to-read-options-for-dump-from-file.patch (60.4K, ../../CAFj8pRAGiNnOUEE5zf3yD60Y82qyLB_ehC=rUNPks0FHULR+oQ@mail.gmail.com/3-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 76f224be5f88b2a33f9bdc4a38a5bd49c86a7d3b Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 110 +++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 489 ++++++++++++++
src/bin/pg_dump/filter.h | 56 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 117 ++++
src/bin/pg_dump/pg_dumpall.c | 61 +-
src/bin/pg_dump/pg_restore.c | 114 ++++
src/bin/pg_dump/t/004_pg_dump_filterfile.pl | 701 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1700 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/004_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index e6b003bf10..2a8cbe41bc 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -829,6 +829,102 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-and-children</option> for table data.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { table | table_and_children | schema | foreign_data | table_data | table_data_and_children } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>, except that
+ it also includes any partitions or inheritance child
+ tables of the table(s) matching the
+ <replaceable class="parameter">pattern</replaceable>.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1169,6 +1265,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1591,6 +1688,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e62d05e5ab..9cad26bbe6 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index eb8f59459a..bff55b6b1a 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -30,6 +30,7 @@ OBJS = \
compress_lz4.o \
compress_none.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -47,8 +48,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..971b3d93eb
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,489 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicelly will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table_data_and_children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table_and_children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ "\"%s\" doesn't support filter for object type \"%s\".",
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or first
+ * char is not alpha. The char '_' is allowed too (exclude first position).
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, "unexpected end of file");
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly way as they are
+ * regenerated.
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, "missing object name pattern");
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude"
+ * object_type can one of: "table", "schema", "foreign_data", "table_data",
+ * "database", "function", "trigger" or "index"
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ "no filter command found (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ "invalid filter command (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, "missing filter object type");
+ return false;
+ }
+
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, "unsupported filter object type: \"%.*s\"", size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..28c5c9c834
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,56 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+
+extern void log_invalid_filter_format(FilterStateData *fstate, char *message);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index ab4c25c781..57562c32f8 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -6,6 +6,7 @@ pg_dump_common_sources = files(
'compress_lz4.c',
'compress_none.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -96,6 +97,7 @@ tests += {
't/001_basic.pl',
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
+ 't/004_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 2e068c6620..6e9475cd8a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -325,6 +326,7 @@ static void appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -403,6 +405,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -655,6 +658,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 15: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1101,6 +1108,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18369,3 +18378,111 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data and children filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" foreign data filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children, objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index cd421c5944..1bfa1c8905 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1908,7 +1916,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1932,3 +1939,55 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * In this moment only excluded databases can be filtered.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" database filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..1ebf573ac4 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,109 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" function filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" type filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" table filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" trigger filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/004_pg_dump_filterfile.pl b/src/bin/pg_dump/t/004_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..ea1702263a
--- /dev/null
+++ b/src/bin/pg_dump/t/004_pg_dump_filterfile.pl
@@ -0,0 +1,701 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 96;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/"exclude" foreign data filter is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !=~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !=~ qr/^COPY public\.bootab/m, "exclude dumped children table");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index e3ffc653e5..fca7b7d0de 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -451,6 +451,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.40.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-03-18 19:48 ` Pavel Stehule <[email protected]>
1 sibling, 0 replies; 19+ messages in thread
From: Pavel Stehule @ 2023-03-18 19:48 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Justin Pryzby <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
fresh rebase
regards
Pavel
Attachments:
[text/x-patch] v20230318-0001-possibility-to-read-options-for-dump-from-file.patch (60.3K, ../../CAFj8pRCKBmXLvLfc3oTQ8JL2hUTx8A1eZgwaDZGnVFOXk3rhhQ@mail.gmail.com/3-v20230318-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 09c64d9a5f2ed033c2691ca25ca6bec23320e1b0 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 110 +++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 489 ++++++++++++++
src/bin/pg_dump/filter.h | 56 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 117 ++++
src/bin/pg_dump/pg_dumpall.c | 61 +-
src/bin/pg_dump/pg_restore.c | 114 ++++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 701 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1700 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d6b1faa804..5672fbb273 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -829,6 +829,102 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-and-children</option> for table data.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { table | table_and_children | schema | foreign_data | table_data | table_data_and_children } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>, except that
+ it also includes any partitions or inheritance child
+ tables of the table(s) matching the
+ <replaceable class="parameter">pattern</replaceable>.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1159,6 +1255,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1581,6 +1678,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..ca2f93652c 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index eb8f59459a..bff55b6b1a 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -30,6 +30,7 @@ OBJS = \
compress_lz4.o \
compress_none.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -47,8 +48,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..971b3d93eb
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,489 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicelly will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table_data_and_children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table_and_children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ "\"%s\" doesn't support filter for object type \"%s\".",
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or first
+ * char is not alpha. The char '_' is allowed too (exclude first position).
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, "unexpected end of file");
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly way as they are
+ * regenerated.
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, "missing object name pattern");
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude"
+ * object_type can one of: "table", "schema", "foreign_data", "table_data",
+ * "database", "function", "trigger" or "index"
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ "no filter command found (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ "invalid filter command (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, "missing filter object type");
+ return false;
+ }
+
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, "unsupported filter object type: \"%.*s\"", size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..28c5c9c834
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,56 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+
+extern void log_invalid_filter_format(FilterStateData *fstate, char *message);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index b2fb7ac77f..0a626e6cc6 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -6,6 +6,7 @@ pg_dump_common_sources = files(
'compress_lz4.c',
'compress_none.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -97,6 +98,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index d62780a088..c7cc4b1573 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -326,6 +327,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -404,6 +406,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -656,6 +659,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 15: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1102,6 +1109,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18475,3 +18484,111 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data and children filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" foreign data filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children, objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index cd421c5944..1bfa1c8905 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1908,7 +1916,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1932,3 +1939,55 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * In this moment only excluded databases can be filtered.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" database filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..1ebf573ac4 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,109 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" function filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" type filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" table filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" trigger filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..ea1702263a
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,701 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 96;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/"exclude" foreign data filter is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !=~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !=~ qr/^COPY public\.bootab/m, "exclude dumped children table");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index e3ffc653e5..fca7b7d0de 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -451,6 +451,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.40.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-03-19 14:01 ` Justin Pryzby <[email protected]>
2023-03-19 20:27 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
1 sibling, 2 replies; 19+ messages in thread
From: Justin Pryzby @ 2023-03-19 14:01 UTC (permalink / raw)
To: Pavel Stehule <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> rebase + enhancing about related option from a563c24
Thanks.
It looks like this doesn't currently handle extensions, which were added
at 6568cef26e.
> + <literal>table_and_children</literal>: tables, works like
> + <option>-t</option>/<option>--table</option>, except that
> + it also includes any partitions or inheritance child
> + tables of the table(s) matching the
> + <replaceable class="parameter">pattern</replaceable>.
Why doesn't this just say "works like --table-and-children" ?
I think as you wrote log_invalid_filter_format(), the messages wouldn't
be translated, because they're printed via %s. One option is to call
_() on the message.
> +ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
!=~ is being interpretted as as numeric "!=" and throwing warnings.
It should be a !~ b, right ?
It'd be nice if perl warnings during the tests were less easy to miss.
> + * char is not alpha. The char '_' is allowed too (exclude first position).
Why is it treated specially? Could it be treated the same as alpha?
> + log_invalid_filter_format(&fstate,
> + "\"include\" table data filter is not allowed");
> + log_invalid_filter_format(&fstate,
> + "\"include\" table data and children filter is not allowed");
For these, it might be better to write the literal option:
> + "include filter for \"table_data_and_children\" is not allowed");
Because the option is a literal and shouldn't be translated.
And it's probably better to write that using %s, like:
> + "include filter for \"%s\" is not allowed");
That makes shorter and fewer strings.
Find attached a bunch of other corrections as 0002.txt
I also dug up what I'd started in november, trying to reduce the code
duplication betwen pg_restore/dump/all. This isn't done, but I might
never finish it, so I'll at least show what I have in case you think
it's a good idea. This passes tests on CI, except for autoconf, due to
using exit_nicely() differently.
--
Justin
Attachments:
[text/x-diff] 0001-possibility-to-read-options-for-dump-from-file.txt (60.4K, ../../[email protected]/2-0001-possibility-to-read-options-for-dump-from-file.txt)
download | inline diff:
From e86d4dca9b4754185a1d559fdea7bf4ee2de8b1a Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH 1/3] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 110 +++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 489 ++++++++++++++
src/bin/pg_dump/filter.h | 56 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 117 ++++
src/bin/pg_dump/pg_dumpall.c | 61 +-
src/bin/pg_dump/pg_restore.c | 114 ++++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 701 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1700 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d6b1faa8042..5672fbb273b 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -829,6 +829,102 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-and-children</option> for table data.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { table | table_and_children | schema | foreign_data | table_data | table_data_and_children } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>, except that
+ it also includes any partitions or inheritance child
+ tables of the table(s) matching the
+ <replaceable class="parameter">pattern</replaceable>.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1159,6 +1255,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1581,6 +1678,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858e..ca2f93652cd 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda06..ffeb564c520 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index eb8f59459a1..bff55b6b1a2 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -30,6 +30,7 @@ OBJS = \
compress_lz4.o \
compress_none.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -47,8 +48,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 00000000000..971b3d93eb1
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,489 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicelly will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table_data_and_children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table_and_children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ "\"%s\" doesn't support filter for object type \"%s\".",
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or first
+ * char is not alpha. The char '_' is allowed too (exclude first position).
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, "unexpected end of file");
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly way as they are
+ * regenerated.
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, "missing object name pattern");
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude"
+ * object_type can one of: "table", "schema", "foreign_data", "table_data",
+ * "database", "function", "trigger" or "index"
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ "no filter command found (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ "invalid filter command (expected \"include\" or \"exclude\")");
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, "missing filter object type");
+ return false;
+ }
+
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, "unsupported filter object type: \"%.*s\"", size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 00000000000..28c5c9c8346
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,56 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+
+extern void log_invalid_filter_format(FilterStateData *fstate, char *message);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index b2fb7ac77fd..0a626e6cc6f 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -6,6 +6,7 @@ pg_dump_common_sources = files(
'compress_lz4.c',
'compress_none.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -97,6 +98,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index d62780a0880..c7cc4b15736 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -326,6 +327,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -404,6 +406,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -656,6 +659,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 15: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1102,6 +1109,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18475,3 +18484,111 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" table data and children filter is not allowed");
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" foreign data filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children, objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index cd421c59443..1bfa1c89051 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1908,7 +1916,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1932,3 +1939,55 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * In this moment only excluded databases can be filtered.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"include\" database filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a1006347..1ebf573ac48 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,109 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifer patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" function filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" type filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" table filter is not allowed");
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" trigger filter is not allowed");
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 00000000000..ea1702263a6
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,701 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 96;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/"exclude" foreign data filter is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !=~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !=~ qr/^COPY public\.bootab/m, "exclude dumped children table");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 0c89c8e5297..0acd9fc89e9 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -437,6 +437,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.34.1
[text/x-diff] 0002-little-fixes.txt (8.0K, ../../[email protected]/3-0002-little-fixes.txt)
download | inline diff:
From c30d5a01d2d78ce87025e8b334e78c5d95937df3 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <[email protected]>
Date: Fri, 17 Mar 2023 22:22:56 -0500
Subject: [PATCH 2/3] little fixes
---
doc/src/sgml/ref/pg_dump.sgml | 2 +-
doc/src/sgml/ref/pg_dumpall.sgml | 2 +-
src/bin/pg_dump/filter.c | 13 +++++++------
src/bin/pg_dump/pg_dump.c | 2 +-
src/bin/pg_dump/pg_dumpall.c | 4 ++--
src/bin/pg_dump/pg_restore.c | 4 ++--
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 8 ++++----
7 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 5672fbb273b..f7c4304aec4 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -843,7 +843,7 @@ PostgreSQL documentation
<option>-n</option>/<option>--schema</option> for schemas,
<option>--include-foreign-data</option> for data on foreign servers and
<option>--exclude-table-data</option>,
- <option>--exclude-table-data-and-and-children</option> for table data.
+ <option>--exclude-table-data-and-children</option> for table data.
To read from <literal>STDIN</literal>, use <filename>-</filename> as the
filename. The <option>--filter</option> option can be specified in
conjunction with the above listed options for including or excluding
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index ca2f93652cd..547fe3803f4 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -127,7 +127,7 @@ PostgreSQL documentation
<listitem>
<para>
Specify a filename from which to read patterns for databases excluded
- from dump. The patterns are interpretted according to the same rules
+ from the dump. The patterns are interpretted according to the same rules
as <option>--exclude-database</option>.
To read from <literal>STDIN</literal>, use <filename>-</filename> as the
filename. The <option>--filter</option> option can be specified in
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
index 971b3d93eb1..41fb31b7253 100644
--- a/src/bin/pg_dump/filter.c
+++ b/src/bin/pg_dump/filter.c
@@ -25,7 +25,7 @@
* Following routines are called from pg_dump, pg_dumpall and pg_restore.
* Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
* different from the one in pg_dumpall, so instead of calling exit_nicely we
- * have to return some error flag (in this case NULL), and exit_nicelly will be
+ * have to return some error flag (in this case NULL), and exit_nicely will be
* executed from caller's routine.
*/
@@ -162,7 +162,7 @@ log_unsupported_filter_object_type(FilterStateData *fstate,
*
* Search for keywords (limited to ascii alphabetic characters) in
* the passed in line buffer. Returns NULL when the buffer is empty or first
- * char is not alpha. The char '_' is allowed too (exclude first position).
+ * char is not alpha. The char '_' is allowed, except as the first character.
* The length of the found keyword is returned in the size parameter.
*/
static const char *
@@ -280,8 +280,9 @@ read_quoted_string(FilterStateData *fstate,
* Note that this function takes special care to sanitize the detected
* identifier (removing extraneous whitespaces or other unnecessary
* characters). This is necessary as most backup/restore filtering functions
- * only recognize identifiers if they are written exactly way as they are
- * regenerated.
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
* Returns a pointer to next character after the found identifier, or NULL on
* error.
*/
@@ -364,8 +365,8 @@ read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
* <command> <object_type> <pattern>
*
* command can be "include" or "exclude"
- * object_type can one of: "table", "schema", "foreign_data", "table_data",
- * "database", "function", "trigger" or "index"
+ * object_type can one of: "table", "table_and_children", "schema", "foreign_data",
+ * "table_data", "table_and_children", "database", "function", "trigger" or "index"
* pattern can be any possibly-quoted and possibly-qualified identifier. It
* follows the same rules as other object include and exclude functions so it
* can also use wildcards.
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index c7cc4b15736..42b8f8d91d5 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -18486,7 +18486,7 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
}
/*
- * read_dump_filters - retrieve object identifer patterns from file
+ * read_dump_filters - retrieve object identifier patterns from file
*
* Parse the specified filter file for include and exclude patterns, and add
* them to the relevant lists. If the filename is "-" then filters will be
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 1bfa1c89051..d58eaaaa063 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -1941,13 +1941,13 @@ hash_string_pointer(char *s)
}
/*
- * read_dumpall_filters - retrieve database identifer patterns from file
+ * read_dumpall_filters - retrieve database identifier patterns from file
*
* Parse the specified filter file for include and exclude patterns, and add
* them to the relevant lists. If the filename is "-" then filters will be
* read from STDIN rather than a file.
*
- * In this moment only excluded databases can be filtered.
+ * At the moment, the only allowed filter is for database exclusion.
*/
static void
read_dumpall_filters(const char *filename, SimpleStringList *pattern)
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 1ebf573ac48..0ea71351737 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -504,7 +504,7 @@ usage(const char *progname)
}
/*
- * read_restore_filters - retrieve object identifer patterns from file
+ * read_restore_filters - retrieve object identifier patterns from file
*
* Parse the specified filter file for include and exclude patterns, and add
* them to the relevant lists. If the filename is "-" then filters will be
@@ -553,7 +553,7 @@ read_restore_filters(const char *filename, RestoreOptions *opts)
else
{
log_invalid_filter_format(&fstate,
- "\"exclude\" type filter is not allowed");
+ "\"exclude\" index filter is not allowed");
break;
}
}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
index ea1702263a6..67314dbd60e 100644
--- a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -640,8 +640,8 @@ command_ok(
$dump = slurp_file($plainfile);
-ok($dump !=~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
-ok($dump !=~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
@@ -680,7 +680,7 @@ command_ok(
$dump = slurp_file($plainfile);
-ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
open $inputfile, '>', "$tempdir/inputfile.txt"
or die "unable to open filterfile for writing";
@@ -698,4 +698,4 @@ command_ok(
$dump = slurp_file($plainfile);
ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
-ok($dump !=~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
--
2.34.1
[text/x-diff] 0003-f-rewrite-using-a-single-filtering-function.txt (41.8K, ../../[email protected]/4-0003-f-rewrite-using-a-single-filtering-function.txt)
download | inline diff:
From 3ea88b30bc696d7e80058e2f1ec1d2d5ad36bd7e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <[email protected]>
Date: Thu, 3 Nov 2022 18:09:14 -0500
Subject: [PATCH 3/3] f!rewrite using a single filtering function
---
src/bin/pg_dump/filter.c | 173 +++++++++++++++++-
src/bin/pg_dump/filter.h | 71 +++++---
src/bin/pg_dump/pg_backup.h | 17 +-
src/bin/pg_dump/pg_backup_archiver.c | 12 +-
src/bin/pg_dump/pg_dump.c | 257 +++++++--------------------
src/bin/pg_dump/pg_dumpall.c | 69 ++-----
src/bin/pg_dump/pg_restore.c | 167 +++++------------
7 files changed, 350 insertions(+), 416 deletions(-)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
index 41fb31b7253..d52f0c7f0bf 100644
--- a/src/bin/pg_dump/filter.c
+++ b/src/bin/pg_dump/filter.c
@@ -15,12 +15,27 @@
#include "common/logging.h"
#include "common/string.h"
#include "filter.h"
+#include "pg_backup.h"
+#include "pg_backup_utils.h"
#include "lib/stringinfo.h"
#include "pqexpbuffer.h"
#define is_keyword_str(cstr, str, bytes) \
((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+
/*
* Following routines are called from pg_dump, pg_dumpall and pg_restore.
* Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
@@ -33,7 +48,7 @@
* Opens filter's file and initialize fstate structure.
* Returns true on success.
*/
-bool
+static bool
filter_init(FilterStateData *fstate, const char *filename)
{
fstate->filename = filename;
@@ -60,7 +75,7 @@ filter_init(FilterStateData *fstate, const char *filename)
/*
* Release allocated resources for the given filter.
*/
-void
+static void
filter_free(FilterStateData *fstate)
{
free(fstate->linebuff.data);
@@ -118,7 +133,7 @@ filter_object_type_name(FilterObjectType fot)
* This is mostly a convenience routine to avoid duplicating file closing code
* in multiple callsites.
*/
-void
+static void
log_invalid_filter_format(FilterStateData *fstate, char *message)
{
if (fstate->fp != stdin)
@@ -136,23 +151,26 @@ log_invalid_filter_format(FilterStateData *fstate, char *message)
fstate->is_error = true;
}
+
/*
* Emit error message "The application doesn't support filter for object type ..."
*
* This is mostly a convenience routine to avoid duplicating file closing code
* in multiple callsites.
*/
-void
+static void
log_unsupported_filter_object_type(FilterStateData *fstate,
const char *appname,
- FilterObjectType fot)
+ FilterObjectType fot,
+ bool is_include)
{
PQExpBuffer str = createPQExpBuffer();
printfPQExpBuffer(str,
- "\"%s\" doesn't support filter for object type \"%s\".",
+ "\"%s\" doesn't support filter for object type \"%s\": %s",
appname,
- filter_object_type_name(fot));
+ filter_object_type_name(fot),
+ is_include ? "include" : "exclude");
log_invalid_filter_format(fstate, str->data);
}
@@ -377,7 +395,7 @@ read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
* error, the function will emit an appropriate error message before returning
* false.
*/
-bool
+static bool
filter_read_item(FilterStateData *fstate,
bool *is_include,
char **objname,
@@ -488,3 +506,142 @@ filter_read_item(FilterStateData *fstate,
return false;
}
+
+/*
+ * read_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+void
+read_filters(const char *appname, const char *filename, struct FilterOpts *filter_opts, struct type_opts *opts, bool *include_everything, int allow_include, int allow_exclude)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1); //
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ /* allows "table and children" whenever table is allowed */
+ if (allow_include & FILTER_OBJECT_TYPE_TABLE)
+ allow_include |= FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ if (allow_include & FILTER_OBJECT_TYPE_TABLE_DATA)
+ allow_include |= FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ if (allow_exclude & FILTER_OBJECT_TYPE_TABLE)
+ allow_exclude |= FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ if (allow_exclude & FILTER_OBJECT_TYPE_TABLE_DATA)
+ allow_exclude |= FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+
+ if (is_include && (allow_include & objtype) == 0)
+ log_unsupported_filter_object_type(&fstate, appname, objtype, is_include);
+ else if (!is_include && (allow_exclude & objtype) == 0)
+ log_unsupported_filter_object_type(&fstate, appname, objtype, is_include);
+ else if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ Assert(!is_include);
+ simple_string_list_append(&filter_opts->database_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ // Assert(is_include);
+ if (!is_include)
+ log_invalid_filter_format(&fstate,
+ "\"exclude\" foreign data filter is not allowed");
+ else
+ simple_string_list_append(&filter_opts->foreign_servers_include_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ Assert(is_include);
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&filter_opts->function_include_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ Assert(is_include);
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&filter_opts->index_include_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&filter_opts->schema_include_patterns, objname);
+ *include_everything = false;
+ }
+ else
+ simple_string_list_append(&filter_opts->schema_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ *include_everything = false;
+ simple_string_list_append(&filter_opts->table_include_patterns, objname);
+ }
+ else
+ simple_string_list_append(&filter_opts->table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ Assert(!is_include);
+ simple_string_list_append(&filter_opts->tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ Assert(!is_include);
+ simple_string_list_append(&filter_opts->tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&filter_opts->table_include_patterns_and_children, objname);
+ *include_everything = false;
+ }
+ else
+ simple_string_list_append(&filter_opts->table_exclude_patterns_and_children, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ Assert(is_include);
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&filter_opts->trigger_include_patterns, objname);
+ }
+ else
+ {
+ // log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ // log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ // log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ pg_fatal("unhandled filter type: %d", objtype);
+ }
+
+ if (objname)
+ free(objname);
+
+ if (fstate.is_error)
+ exit_nicely(1); //
+ }
+
+ if (fstate.is_error)
+ exit_nicely(1); //
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
index 28c5c9c8346..99a2a5a9a07 100644
--- a/src/bin/pg_dump/filter.h
+++ b/src/bin/pg_dump/filter.h
@@ -14,43 +14,56 @@
#define FILTER_H
#include "lib/stringinfo.h"
-
-/*
- * State data for reading filter items from stream
- */
-typedef struct
-{
- FILE *fp;
- const char *filename;
- int lineno;
- StringInfoData linebuff;
- bool is_error;
-} FilterStateData;
+#include "fe_utils/simple_list.h"
/*
* List of objects that can be specified in filter file
*/
typedef enum
{
- FILTER_OBJECT_TYPE_NONE,
- FILTER_OBJECT_TYPE_TABLE_DATA,
- FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
- FILTER_OBJECT_TYPE_DATABASE,
- FILTER_OBJECT_TYPE_FOREIGN_DATA,
- FILTER_OBJECT_TYPE_FUNCTION,
- FILTER_OBJECT_TYPE_INDEX,
- FILTER_OBJECT_TYPE_SCHEMA,
- FILTER_OBJECT_TYPE_TABLE,
- FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
- FILTER_OBJECT_TYPE_TRIGGER
+ FILTER_OBJECT_TYPE_NONE = 1<<0,
+ FILTER_OBJECT_TYPE_TABLE_DATA = 1<<1,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN = 1<<2,
+ FILTER_OBJECT_TYPE_DATABASE = 1<<3,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA = 1<<4,
+ FILTER_OBJECT_TYPE_FUNCTION = 1<<5,
+ FILTER_OBJECT_TYPE_INDEX = 1<<6,
+ FILTER_OBJECT_TYPE_SCHEMA = 1<<7,
+ FILTER_OBJECT_TYPE_TABLE = 1<<8,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN = 1<<9,
+ FILTER_OBJECT_TYPE_TRIGGER = 1<<10,
} FilterObjectType;
-extern bool filter_init(FilterStateData *fstate, const char *filename);
-extern void filter_free(FilterStateData *fstate);
+struct FilterOpts {
+ SimpleStringList schema_include_patterns;
+ SimpleOidList schema_include_oids;
+ SimpleStringList schema_exclude_patterns;
+ SimpleOidList schema_exclude_oids;
+
+ SimpleStringList table_include_patterns;
+ SimpleStringList table_include_patterns_and_children;
+ SimpleOidList table_include_oids;
+ SimpleStringList table_exclude_patterns;
+ SimpleStringList table_exclude_patterns_and_children;
+ SimpleOidList table_exclude_oids;
+ SimpleStringList tabledata_exclude_patterns;
+ SimpleStringList tabledata_exclude_patterns_and_children;
+ SimpleOidList tabledata_exclude_oids;
+ SimpleStringList foreign_servers_include_patterns;
+ SimpleOidList foreign_servers_include_oids;
+
+ SimpleStringList extension_include_patterns;
+ SimpleOidList extension_include_oids;
+
+ SimpleStringList database_exclude_patterns;
+ SimpleStringList index_include_patterns;
+ SimpleStringList function_include_patterns;
+ SimpleStringList trigger_include_patterns;
+};
-extern void log_invalid_filter_format(FilterStateData *fstate, char *message);
-extern void log_unsupported_filter_object_type(FilterStateData *fstate,
- const char *appname, FilterObjectType fot);
-extern bool filter_read_item(FilterStateData *fstate, bool *is_include, char **objname, FilterObjectType *objtype);
+struct type_opts; /* fwd decl */
+extern void read_filters(const char *appname, const char *filename, struct FilterOpts *filteropts,
+ struct type_opts *opts, bool *include_everything,
+ int allow_include, int allow_exclude);
#endif
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index aba780ef4b1..0794ec3a301 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -26,6 +26,7 @@
#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
+#include "filter.h"
typedef enum trivalue
@@ -90,6 +91,15 @@ typedef struct _connParams
char *override_dbname;
} ConnParams;
+struct type_opts
+{
+ int selTypes;
+ int selIndex;
+ int selFunction;
+ int selTrigger;
+ int selTable;
+};
+
typedef struct _restoreOptions
{
int createDB; /* Issue commands to create the database */
@@ -127,11 +137,8 @@ typedef struct _restoreOptions
int format;
char *formatName;
- int selTypes;
- int selIndex;
- int selFunction;
- int selTrigger;
- int selTable;
+ struct type_opts typeopts;
+
SimpleStringList indexNames;
SimpleStringList functionNames;
SimpleStringList schemaNames;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 3337d34e405..57e73bd608d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -2890,7 +2890,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
else if (ropt->schemaNames.head != NULL ||
ropt->schemaExcludeNames.head != NULL ||
- ropt->selTypes)
+ ropt->typeopts.selTypes)
{
/*
* In a selective dump/restore, we want to restore these dependent
@@ -2930,7 +2930,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
simple_string_list_member(&ropt->schemaExcludeNames, te->namespace))
return 0;
- if (ropt->selTypes)
+ if (ropt->typeopts.selTypes)
{
if (strcmp(te->desc, "TABLE") == 0 ||
strcmp(te->desc, "TABLE DATA") == 0 ||
@@ -2941,7 +2941,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "SEQUENCE") == 0 ||
strcmp(te->desc, "SEQUENCE SET") == 0)
{
- if (!ropt->selTable)
+ if (!ropt->typeopts.selTable)
return 0;
if (ropt->tableNames.head != NULL &&
!simple_string_list_member(&ropt->tableNames, te->tag))
@@ -2949,7 +2949,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
else if (strcmp(te->desc, "INDEX") == 0)
{
- if (!ropt->selIndex)
+ if (!ropt->typeopts.selIndex)
return 0;
if (ropt->indexNames.head != NULL &&
!simple_string_list_member(&ropt->indexNames, te->tag))
@@ -2959,7 +2959,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "AGGREGATE") == 0 ||
strcmp(te->desc, "PROCEDURE") == 0)
{
- if (!ropt->selFunction)
+ if (!ropt->typeopts.selFunction)
return 0;
if (ropt->functionNames.head != NULL &&
!simple_string_list_member(&ropt->functionNames, te->tag))
@@ -2967,7 +2967,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
else if (strcmp(te->desc, "TRIGGER") == 0)
{
- if (!ropt->selTrigger)
+ if (!ropt->typeopts.selTrigger)
return 0;
if (ropt->triggerNames.head != NULL &&
!simple_string_list_member(&ropt->triggerNames, te->tag))
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 42b8f8d91d5..b4f1ccb2f9b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -114,26 +114,7 @@ static pg_compress_algorithm compression_algorithm = PG_COMPRESSION_NONE;
* The string lists record the patterns given by command-line switches,
* which we then convert to lists of OIDs of matching objects.
*/
-static SimpleStringList schema_include_patterns = {NULL, NULL};
-static SimpleOidList schema_include_oids = {NULL, NULL};
-static SimpleStringList schema_exclude_patterns = {NULL, NULL};
-static SimpleOidList schema_exclude_oids = {NULL, NULL};
-
-static SimpleStringList table_include_patterns = {NULL, NULL};
-static SimpleStringList table_include_patterns_and_children = {NULL, NULL};
-static SimpleOidList table_include_oids = {NULL, NULL};
-static SimpleStringList table_exclude_patterns = {NULL, NULL};
-static SimpleStringList table_exclude_patterns_and_children = {NULL, NULL};
-static SimpleOidList table_exclude_oids = {NULL, NULL};
-static SimpleStringList tabledata_exclude_patterns = {NULL, NULL};
-static SimpleStringList tabledata_exclude_patterns_and_children = {NULL, NULL};
-static SimpleOidList tabledata_exclude_oids = {NULL, NULL};
-
-static SimpleStringList foreign_servers_include_patterns = {NULL, NULL};
-static SimpleOidList foreign_servers_include_oids = {NULL, NULL};
-
-static SimpleStringList extension_include_patterns = {NULL, NULL};
-static SimpleOidList extension_include_oids = {NULL, NULL};
+struct FilterOpts filter_opts;
static const CatalogId nilCatalogId = {0, 0};
@@ -327,7 +308,6 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
-static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -495,7 +475,7 @@ main(int argc, char **argv)
break;
case 'e': /* include extension(s) */
- simple_string_list_append(&extension_include_patterns, optarg);
+ simple_string_list_append(&filter_opts.extension_include_patterns, optarg);
dopt.include_everything = false;
break;
@@ -523,12 +503,12 @@ main(int argc, char **argv)
break;
case 'n': /* include schema(s) */
- simple_string_list_append(&schema_include_patterns, optarg);
+ simple_string_list_append(&filter_opts.schema_include_patterns, optarg);
dopt.include_everything = false;
break;
case 'N': /* exclude schema(s) */
- simple_string_list_append(&schema_exclude_patterns, optarg);
+ simple_string_list_append(&filter_opts.schema_exclude_patterns, optarg);
break;
case 'O': /* Don't reconnect to match owner */
@@ -552,12 +532,12 @@ main(int argc, char **argv)
break;
case 't': /* include table(s) */
- simple_string_list_append(&table_include_patterns, optarg);
+ simple_string_list_append(&filter_opts.table_include_patterns, optarg);
dopt.include_everything = false;
break;
case 'T': /* exclude table(s) */
- simple_string_list_append(&table_exclude_patterns, optarg);
+ simple_string_list_append(&filter_opts.table_exclude_patterns, optarg);
break;
case 'U':
@@ -600,7 +580,7 @@ main(int argc, char **argv)
break;
case 4: /* exclude table(s) data */
- simple_string_list_append(&tabledata_exclude_patterns, optarg);
+ simple_string_list_append(&filter_opts.tabledata_exclude_patterns, optarg);
break;
case 5: /* section */
@@ -639,28 +619,35 @@ main(int argc, char **argv)
break;
case 11: /* include foreign data */
- simple_string_list_append(&foreign_servers_include_patterns,
+ simple_string_list_append(&filter_opts.foreign_servers_include_patterns,
optarg);
break;
case 12: /* include table(s) and their children */
- simple_string_list_append(&table_include_patterns_and_children,
+ simple_string_list_append(&filter_opts.table_include_patterns_and_children,
optarg);
dopt.include_everything = false;
break;
case 13: /* exclude table(s) and their children */
- simple_string_list_append(&table_exclude_patterns_and_children,
+ simple_string_list_append(&filter_opts.table_exclude_patterns_and_children,
optarg);
break;
case 14: /* exclude data of table(s) and children */
- simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ simple_string_list_append(&filter_opts.tabledata_exclude_patterns_and_children,
optarg);
break;
case 15: /* object filters from file */
- read_dump_filters(optarg, &dopt);
+ {
+ struct type_opts typeopts;
+ // XXX: do something with it....
+ read_filters(progname, optarg, &filter_opts, &typeopts, &dopt.include_everything,
+ FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_FOREIGN_DATA, /* inclusions */
+ FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_TABLE_DATA | /*XXX not really:*/FILTER_OBJECT_TYPE_FOREIGN_DATA /* exclusions */
+ );
+ }
break;
default:
@@ -701,10 +688,10 @@ main(int argc, char **argv)
if (dopt.dataOnly && dopt.schemaOnly)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
- if (dopt.schemaOnly && foreign_servers_include_patterns.head != NULL)
+ if (dopt.schemaOnly && filter_opts.foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
- if (numWorkers > 1 && foreign_servers_include_patterns.head != NULL)
+ if (numWorkers > 1 && filter_opts.foreign_servers_include_patterns.head != NULL)
pg_fatal("option --include-foreign-data is not supported with parallel backup");
if (dopt.dataOnly && dopt.outputClean)
@@ -828,57 +815,59 @@ main(int argc, char **argv)
pg_log_info("last built-in OID is %u", g_last_builtin_oid);
/* Expand schema selection patterns into OID lists */
- if (schema_include_patterns.head != NULL)
+ if (filter_opts.schema_include_patterns.head != NULL)
{
- expand_schema_name_patterns(fout, &schema_include_patterns,
- &schema_include_oids,
+ expand_schema_name_patterns(fout, &filter_opts.schema_include_patterns,
+ &filter_opts.schema_include_oids,
strict_names);
- if (schema_include_oids.head == NULL)
+ if (filter_opts.schema_include_oids.head == NULL)
pg_fatal("no matching schemas were found");
}
- expand_schema_name_patterns(fout, &schema_exclude_patterns,
- &schema_exclude_oids,
+ expand_schema_name_patterns(fout, &filter_opts.schema_exclude_patterns,
+ &filter_opts.schema_exclude_oids,
false);
/* non-matching exclusion patterns aren't an error */
/* Expand table selection patterns into OID lists */
- expand_table_name_patterns(fout, &table_include_patterns,
- &table_include_oids,
+ expand_table_name_patterns(fout, &filter_opts.table_include_patterns,
+ &filter_opts.table_include_oids,
strict_names, false);
- expand_table_name_patterns(fout, &table_include_patterns_and_children,
- &table_include_oids,
+
+ expand_table_name_patterns(fout, &filter_opts.table_include_patterns_and_children,
+ &filter_opts.table_include_oids,
strict_names, true);
- if ((table_include_patterns.head != NULL ||
- table_include_patterns_and_children.head != NULL) &&
- table_include_oids.head == NULL)
+
+ if ((filter_opts.table_include_patterns.head != NULL ||
+ filter_opts.table_include_patterns_and_children.head != NULL) &&
+ filter_opts.table_include_oids.head == NULL)
pg_fatal("no matching tables were found");
- expand_table_name_patterns(fout, &table_exclude_patterns,
- &table_exclude_oids,
+ expand_table_name_patterns(fout, &filter_opts.table_exclude_patterns,
+ &filter_opts.table_exclude_oids,
false, false);
- expand_table_name_patterns(fout, &table_exclude_patterns_and_children,
- &table_exclude_oids,
+ expand_table_name_patterns(fout, &filter_opts.table_exclude_patterns_and_children,
+ &filter_opts.table_exclude_oids,
false, true);
- expand_table_name_patterns(fout, &tabledata_exclude_patterns,
- &tabledata_exclude_oids,
+ expand_table_name_patterns(fout, &filter_opts.tabledata_exclude_patterns,
+ &filter_opts.tabledata_exclude_oids,
false, false);
- expand_table_name_patterns(fout, &tabledata_exclude_patterns_and_children,
- &tabledata_exclude_oids,
+ expand_table_name_patterns(fout, &filter_opts.tabledata_exclude_patterns_and_children,
+ &filter_opts.tabledata_exclude_oids,
false, true);
- expand_foreign_server_name_patterns(fout, &foreign_servers_include_patterns,
- &foreign_servers_include_oids);
+ expand_foreign_server_name_patterns(fout, &filter_opts.foreign_servers_include_patterns,
+ &filter_opts.foreign_servers_include_oids);
/* non-matching exclusion patterns aren't an error */
/* Expand extension selection patterns into OID lists */
- if (extension_include_patterns.head != NULL)
+ if (filter_opts.extension_include_patterns.head != NULL)
{
- expand_extension_name_patterns(fout, &extension_include_patterns,
- &extension_include_oids,
+ expand_extension_name_patterns(fout, &filter_opts.extension_include_patterns,
+ &filter_opts.extension_include_oids,
strict_names);
- if (extension_include_oids.head == NULL)
+ if (filter_opts.extension_include_oids.head == NULL)
pg_fatal("no matching extensions were found");
}
@@ -1729,11 +1718,11 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout)
* namespaces. If specific namespaces are being dumped, dump just those
* namespaces. Otherwise, dump all non-system namespaces.
*/
- if (table_include_oids.head != NULL)
+ if (filter_opts.table_include_oids.head != NULL)
nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE;
- else if (schema_include_oids.head != NULL)
+ else if (filter_opts.schema_include_oids.head != NULL)
nsinfo->dobj.dump_contains = nsinfo->dobj.dump =
- simple_oid_list_member(&schema_include_oids,
+ simple_oid_list_member(&filter_opts.schema_include_oids,
nsinfo->dobj.catId.oid) ?
DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE;
else if (fout->remoteVersion >= 90600 &&
@@ -1782,7 +1771,7 @@ selectDumpableNamespace(NamespaceInfo *nsinfo, Archive *fout)
* In any case, a namespace can be excluded by an exclusion switch
*/
if (nsinfo->dobj.dump_contains &&
- simple_oid_list_member(&schema_exclude_oids,
+ simple_oid_list_member(&filter_opts.schema_exclude_oids,
nsinfo->dobj.catId.oid))
nsinfo->dobj.dump_contains = nsinfo->dobj.dump = DUMP_COMPONENT_NONE;
@@ -1810,8 +1799,8 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout)
* If specific tables are being dumped, dump just those tables; else, dump
* according to the parent namespace's dump flag.
*/
- if (table_include_oids.head != NULL)
- tbinfo->dobj.dump = simple_oid_list_member(&table_include_oids,
+ if (filter_opts.table_include_oids.head != NULL)
+ tbinfo->dobj.dump = simple_oid_list_member(&filter_opts.table_include_oids,
tbinfo->dobj.catId.oid) ?
DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE;
else
@@ -1821,7 +1810,7 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout)
* In any case, a table can be excluded by an exclusion switch
*/
if (tbinfo->dobj.dump &&
- simple_oid_list_member(&table_exclude_oids,
+ simple_oid_list_member(&filter_opts.table_exclude_oids,
tbinfo->dobj.catId.oid))
tbinfo->dobj.dump = DUMP_COMPONENT_NONE;
}
@@ -2005,9 +1994,9 @@ selectDumpableExtension(ExtensionInfo *extinfo, DumpOptions *dopt)
else
{
/* check if there is a list of extensions to dump */
- if (extension_include_oids.head != NULL)
+ if (filter_opts.extension_include_oids.head != NULL)
extinfo->dobj.dump = extinfo->dobj.dump_contains =
- simple_oid_list_member(&extension_include_oids,
+ simple_oid_list_member(&filter_opts.extension_include_oids,
extinfo->dobj.catId.oid) ?
DUMP_COMPONENT_ALL : DUMP_COMPONENT_NONE;
else
@@ -2719,8 +2708,8 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo)
return;
/* Skip FOREIGN TABLEs (no data to dump) unless requested explicitly */
if (tbinfo->relkind == RELKIND_FOREIGN_TABLE &&
- (foreign_servers_include_oids.head == NULL ||
- !simple_oid_list_member(&foreign_servers_include_oids,
+ (filter_opts.foreign_servers_include_oids.head == NULL ||
+ !simple_oid_list_member(&filter_opts.foreign_servers_include_oids,
tbinfo->foreign_server)))
return;
/* Skip partitioned tables (data in partitions) */
@@ -2733,7 +2722,7 @@ makeTableDataInfo(DumpOptions *dopt, TableInfo *tbinfo)
return;
/* Check that the data is not explicitly excluded */
- if (simple_oid_list_member(&tabledata_exclude_oids,
+ if (simple_oid_list_member(&filter_opts.tabledata_exclude_oids,
tbinfo->dobj.catId.oid))
return;
@@ -17849,8 +17838,8 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[],
* Check if this extension is listed as to include in the dump. If
* not, any table data associated with it is discarded.
*/
- if (extension_include_oids.head != NULL &&
- !simple_oid_list_member(&extension_include_oids,
+ if (filter_opts.extension_include_oids.head != NULL &&
+ !simple_oid_list_member(&filter_opts.extension_include_oids,
curext->dobj.catId.oid))
continue;
@@ -17883,8 +17872,8 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[],
if (!(curext->dobj.dump & DUMP_COMPONENT_DEFINITION))
{
/* check table explicitly requested */
- if (table_include_oids.head != NULL &&
- simple_oid_list_member(&table_include_oids,
+ if (filter_opts.table_include_oids.head != NULL &&
+ simple_oid_list_member(&filter_opts.table_include_oids,
configtbloid))
dumpobj = true;
@@ -17895,13 +17884,13 @@ processExtensionTables(Archive *fout, ExtensionInfo extinfo[],
}
/* check table excluded by an exclusion switch */
- if (table_exclude_oids.head != NULL &&
- simple_oid_list_member(&table_exclude_oids,
+ if (filter_opts.table_exclude_oids.head != NULL &&
+ simple_oid_list_member(&filter_opts.table_exclude_oids,
configtbloid))
dumpobj = false;
/* check schema excluded by an exclusion switch */
- if (simple_oid_list_member(&schema_exclude_oids,
+ if (simple_oid_list_member(&filter_opts.schema_exclude_oids,
configtbl->dobj.namespace->dobj.catId.oid))
dumpobj = false;
@@ -18484,111 +18473,3 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
-
-/*
- * read_dump_filters - retrieve object identifier patterns from file
- *
- * Parse the specified filter file for include and exclude patterns, and add
- * them to the relevant lists. If the filename is "-" then filters will be
- * read from STDIN rather than a file.
- */
-static void
-read_dump_filters(const char *filename, DumpOptions *dopt)
-{
- FilterStateData fstate;
- bool is_include;
- char *objname;
- FilterObjectType objtype;
-
- if (!filter_init(&fstate, filename))
- exit_nicely(1);
-
- while (filter_read_item(&fstate, &is_include, &objname, &objtype))
- {
- /* ignore comments and empty lines */
- if (objtype == FILTER_OBJECT_TYPE_NONE)
- continue;
-
- if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
- {
- if (is_include)
- {
- log_invalid_filter_format(&fstate,
- "\"include\" table data filter is not allowed");
- break;
- }
- else
- simple_string_list_append(&tabledata_exclude_patterns,
- objname);
- }
- else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
- {
- if (is_include)
- {
- log_invalid_filter_format(&fstate,
- "\"include\" table data and children filter is not allowed");
- break;
- }
- else
- simple_string_list_append(&tabledata_exclude_patterns_and_children,
- objname);
- }
- else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
- {
- if (is_include)
- simple_string_list_append(&foreign_servers_include_patterns,
- objname);
- else
- {
- log_invalid_filter_format(&fstate,
- "\"exclude\" foreign data filter is not allowed");
- break;
- }
- }
- else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
- {
- if (is_include)
- {
- simple_string_list_append(&schema_include_patterns,
- objname);
- dopt->include_everything = false;
- }
- else
- simple_string_list_append(&schema_exclude_patterns,
- objname);
- }
- else if (objtype == FILTER_OBJECT_TYPE_TABLE)
- {
- if (is_include)
- {
- simple_string_list_append(&table_include_patterns, objname);
- dopt->include_everything = false;
- }
- else
- simple_string_list_append(&table_exclude_patterns, objname);
- }
- else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
- {
- if (is_include)
- {
- simple_string_list_append(&table_include_patterns_and_children, objname);
- dopt->include_everything = false;
- }
- else
- simple_string_list_append(&table_exclude_patterns_and_children, objname);
- }
- else
- {
- log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
- break;
- }
-
- if (objname)
- free(objname);
- }
-
- filter_free(&fstate);
-
- if (fstate.is_error)
- exit_nicely(1);
-}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index d58eaaaa063..502ab128337 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -82,7 +82,6 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
-static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -120,8 +119,8 @@ static char role_catalog[10];
static FILE *OPF;
static char *filename = NULL;
-static SimpleStringList database_exclude_patterns = {NULL, NULL};
static SimpleStringList database_exclude_names = {NULL, NULL};
+struct FilterOpts filter_opts;
#define exit_nicely(code) exit(code)
@@ -355,7 +354,7 @@ main(int argc, char *argv[])
break;
case 6:
- simple_string_list_append(&database_exclude_patterns, optarg);
+ simple_string_list_append(&filter_opts.database_exclude_patterns, optarg);
break;
case 7:
@@ -364,7 +363,12 @@ main(int argc, char *argv[])
break;
case 8:
- read_dumpall_filters(optarg, &database_exclude_patterns);
+ {
+ bool foo;
+ struct type_opts typeopts;
+ read_filters(progname, optarg, &filter_opts, &typeopts, &foo,
+ 0, FILTER_OBJECT_TYPE_DATABASE);
+ }
break;
default:
@@ -383,7 +387,8 @@ main(int argc, char *argv[])
exit_nicely(1);
}
- if (database_exclude_patterns.head != NULL &&
+ if (filter_opts.database_exclude_patterns.head != NULL &&
+
(globals_only || roles_only || tablespaces_only))
{
pg_log_error("option --exclude-database cannot be used together with -g/--globals-only, -r/--roles-only, or -t/--tablespaces-only");
@@ -495,7 +500,7 @@ main(int argc, char *argv[])
/*
* Get a list of database names that match the exclude patterns
*/
- expand_dbname_patterns(conn, &database_exclude_patterns,
+ expand_dbname_patterns(conn, &filter_opts.database_exclude_patterns,
&database_exclude_names);
/*
@@ -1939,55 +1944,3 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
-
-/*
- * read_dumpall_filters - retrieve database identifier patterns from file
- *
- * Parse the specified filter file for include and exclude patterns, and add
- * them to the relevant lists. If the filename is "-" then filters will be
- * read from STDIN rather than a file.
- *
- * At the moment, the only allowed filter is for database exclusion.
- */
-static void
-read_dumpall_filters(const char *filename, SimpleStringList *pattern)
-{
- FilterStateData fstate;
- bool is_include;
- char *objname;
- FilterObjectType objtype;
-
- if (!filter_init(&fstate, filename))
- exit_nicely(1);
-
- while (filter_read_item(&fstate, &is_include, &objname, &objtype))
- {
- if (objtype == FILTER_OBJECT_TYPE_NONE)
- continue;
-
- if (objtype == FILTER_OBJECT_TYPE_DATABASE)
- {
- if (!is_include)
- simple_string_list_append(pattern, objname);
- else
- {
- log_invalid_filter_format(&fstate,
- "\"include\" database filter is not allowed");
- break;
- }
- }
- else
- {
- log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
- break;
- }
-
- if (objname)
- free(objname);
- }
-
- filter_free(&fstate);
-
- if (fstate.is_error)
- exit_nicely(1);
-}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 0ea71351737..02390886946 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -53,12 +53,12 @@
#include "pg_backup_utils.h"
static void usage(const char *progname);
-static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
{
RestoreOptions *opts;
+ struct FilterOpts filter_opts = {0};
int c;
int exit_code;
int numWorkers = 1;
@@ -202,11 +202,11 @@ main(int argc, char **argv)
break;
case 'n': /* Dump data for this schema only */
- simple_string_list_append(&opts->schemaNames, optarg);
+ simple_string_list_append(&filter_opts.schema_include_patterns, optarg);
break;
case 'N': /* Do not dump data for this schema */
- simple_string_list_append(&opts->schemaExcludeNames, optarg);
+ simple_string_list_append(&filter_opts.schema_exclude_patterns, optarg);
break;
case 'O':
@@ -221,19 +221,19 @@ main(int argc, char **argv)
/* no-op, still accepted for backwards compatibility */
break;
case 'P': /* Function */
- opts->selTypes = 1;
- opts->selFunction = 1;
- simple_string_list_append(&opts->functionNames, optarg);
+ opts->typeopts.selTypes = 1;
+ opts->typeopts.selFunction = 1;
+ simple_string_list_append(&filter_opts.function_include_patterns, optarg);
break;
case 'I': /* Index */
- opts->selTypes = 1;
- opts->selIndex = 1;
- simple_string_list_append(&opts->indexNames, optarg);
+ opts->typeopts.selTypes = 1;
+ opts->typeopts.selIndex = 1;
+ simple_string_list_append(&filter_opts.index_include_patterns, optarg);
break;
case 'T': /* Trigger */
- opts->selTypes = 1;
- opts->selTrigger = 1;
- simple_string_list_append(&opts->triggerNames, optarg);
+ opts->typeopts.selTypes = 1;
+ opts->typeopts.selTrigger = 1;
+ simple_string_list_append(&filter_opts.trigger_include_patterns, optarg);
break;
case 's': /* dump schema only */
opts->schemaOnly = 1;
@@ -243,9 +243,9 @@ main(int argc, char **argv)
opts->superuser = pg_strdup(optarg);
break;
case 't': /* Dump specified table(s) only */
- opts->selTypes = 1;
- opts->selTable = 1;
- simple_string_list_append(&opts->tableNames, optarg);
+ opts->typeopts.selTypes = 1;
+ opts->typeopts.selTable = 1;
+ simple_string_list_append(&filter_opts.table_include_patterns, optarg);
break;
case 'U':
@@ -290,7 +290,14 @@ main(int argc, char **argv)
break;
case 4:
- read_restore_filters(optarg, opts);
+ {
+ bool include_everything = opts->include_everything;
+ read_filters(progname, optarg, &filter_opts, &opts->typeopts, &include_everything,
+ FILTER_OBJECT_TYPE_SCHEMA | FILTER_OBJECT_TYPE_TABLE_DATA | FILTER_OBJECT_TYPE_FOREIGN_DATA | FILTER_OBJECT_TYPE_FUNCTION | FILTER_OBJECT_TYPE_INDEX | FILTER_OBJECT_TYPE_TABLE | FILTER_OBJECT_TYPE_TRIGGER, /* allowed to include */
+ FILTER_OBJECT_TYPE_SCHEMA /* allowed to exclude */
+ );
+ opts->include_everything = include_everything;
+ }
break;
default:
@@ -300,6 +307,28 @@ main(int argc, char **argv)
}
}
+ /*
+ * Any values read into filter_opts are appended to the corresponding
+ * values in opts, which are used during restore.
+ */
+ for (SimpleStringListCell *cell = filter_opts.trigger_include_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->triggerNames, cell->val);
+
+ for (SimpleStringListCell *cell = filter_opts.schema_include_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->schemaNames, cell->val);
+
+ for (SimpleStringListCell *cell = filter_opts.schema_exclude_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->schemaExcludeNames, cell->val);
+
+ for (SimpleStringListCell *cell = filter_opts.function_include_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->functionNames, cell->val);
+
+ for (SimpleStringListCell *cell = filter_opts.index_include_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->indexNames, cell->val);
+
+ for (SimpleStringListCell *cell = filter_opts.table_include_patterns.head; cell; cell = cell->next)
+ simple_string_list_append(&opts->tableNames, cell->val);
+
/* Get file name from command line */
if (optind < argc)
inputFileSpec = argv[optind++];
@@ -502,109 +531,3 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
-
-/*
- * read_restore_filters - retrieve object identifier patterns from file
- *
- * Parse the specified filter file for include and exclude patterns, and add
- * them to the relevant lists. If the filename is "-" then filters will be
- * read from STDIN rather than a file.
- */
-static void
-read_restore_filters(const char *filename, RestoreOptions *opts)
-{
- FilterStateData fstate;
- bool is_include;
- char *objname;
- FilterObjectType objtype;
-
- if (!filter_init(&fstate, filename))
- exit_nicely(1);
-
- while (filter_read_item(&fstate, &is_include, &objname, &objtype))
- {
- /* ignore comments or empty lines */
- if (objtype == FILTER_OBJECT_TYPE_NONE)
- continue;
-
- if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
- {
- if (is_include)
- {
- opts->selTypes = 1;
- opts->selFunction = 1;
- simple_string_list_append(&opts->functionNames, objname);
- }
- else
- {
- log_invalid_filter_format(&fstate,
- "\"exclude\" function filter is not allowed");
- break;
- }
- }
- else if (objtype == FILTER_OBJECT_TYPE_INDEX)
- {
- if (is_include)
- {
- opts->selTypes = 1;
- opts->selIndex = 1;
- simple_string_list_append(&opts->indexNames, objname);
- }
- else
- {
- log_invalid_filter_format(&fstate,
- "\"exclude\" index filter is not allowed");
- break;
- }
- }
- else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
- {
- if (is_include)
- simple_string_list_append(&opts->schemaNames, objname);
- else
- simple_string_list_append(&opts->schemaExcludeNames, objname);
- }
- else if (objtype == FILTER_OBJECT_TYPE_TABLE)
- {
- if (is_include)
- {
- opts->selTypes = 1;
- opts->selTable = 1;
- simple_string_list_append(&opts->tableNames, objname);
- }
- else
- {
- log_invalid_filter_format(&fstate,
- "\"exclude\" table filter is not allowed");
- break;
- }
- }
- else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
- {
- if (is_include)
- {
- opts->selTypes = 1;
- opts->selTrigger = 1;
- simple_string_list_append(&opts->triggerNames, objname);
- }
- else
- {
- log_invalid_filter_format(&fstate,
- "\"exclude\" trigger filter is not allowed");
- break;
- }
- }
- else
- {
- log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
- break;
- }
-
- if (objname)
- free(objname);
- }
-
- filter_free(&fstate);
- if (fstate.is_error)
- exit_nicely(1);
-}
--
2.34.1
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
@ 2023-03-19 20:27 ` Pavel Stehule <[email protected]>
1 sibling, 0 replies; 19+ messages in thread
From: Pavel Stehule @ 2023-03-19 20:27 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
ne 19. 3. 2023 v 15:01 odesílatel Justin Pryzby <[email protected]>
napsal:
> On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> > rebase + enhancing about related option from a563c24
>
> Thanks.
>
> It looks like this doesn't currently handle extensions, which were added
> at 6568cef26e.
>
> > + <literal>table_and_children</literal>: tables, works like
> > + <option>-t</option>/<option>--table</option>, except that
> > + it also includes any partitions or inheritance child
> > + tables of the table(s) matching the
> > + <replaceable class="parameter">pattern</replaceable>.
>
> Why doesn't this just say "works like --table-and-children" ?
>
> I think as you wrote log_invalid_filter_format(), the messages wouldn't
> be translated, because they're printed via %s. One option is to call
> _() on the message.
>
> > +ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped
> children table");
>
> !=~ is being interpretted as as numeric "!=" and throwing warnings.
> It should be a !~ b, right ?
> It'd be nice if perl warnings during the tests were less easy to miss.
>
> > + * char is not alpha. The char '_' is allowed too (exclude first
> position).
>
> Why is it treated specially? Could it be treated the same as alpha?
>
> > + log_invalid_filter_format(&fstate,
> > +
> "\"include\" table data filter is not allowed");
> > + log_invalid_filter_format(&fstate,
> > +
> "\"include\" table data and children filter is not allowed");
>
> For these, it might be better to write the literal option:
>
> > +
> "include filter for \"table_data_and_children\" is not allowed");
>
> Because the option is a literal and shouldn't be translated.
> And it's probably better to write that using %s, like:
>
> > +
> "include filter for \"%s\" is not allowed");
>
> That makes shorter and fewer strings.
>
> Find attached a bunch of other corrections as 0002.txt
>
Thank you very much - I'll recheck the mentioned points tomorrow.
>
> I also dug up what I'd started in november, trying to reduce the code
> duplication betwen pg_restore/dump/all. This isn't done, but I might
> never finish it, so I'll at least show what I have in case you think
> it's a good idea. This passes tests on CI, except for autoconf, due to
> using exit_nicely() differently.
>
Your implementation reduced 60 lines, but the interface and code is more
complex. I cannot say what is significantly better. Personally, in this
case, I prefer my variant, because I think it is a little bit more
readable, and possible modification can be more simple. But this is just my
opinion, and I have no problem accepting other opinions. I can imagine to
define some configuration array like getopt, but it looks like over
engineering
Regards
Pavel
> --
> Justin
>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
@ 2023-03-20 07:01 ` Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-03-20 07:01 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
ne 19. 3. 2023 v 15:01 odesílatel Justin Pryzby <[email protected]>
napsal:
> On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> > rebase + enhancing about related option from a563c24
>
> Thanks.
>
> It looks like this doesn't currently handle extensions, which were added
> at 6568cef26e.
>
> > + <literal>table_and_children</literal>: tables, works like
> > + <option>-t</option>/<option>--table</option>, except that
> > + it also includes any partitions or inheritance child
> > + tables of the table(s) matching the
> > + <replaceable class="parameter">pattern</replaceable>.
>
> Why doesn't this just say "works like --table-and-children" ?
>
changed
>
> I think as you wrote log_invalid_filter_format(), the messages wouldn't
> be translated, because they're printed via %s. One option is to call
> _() on the message.
>
fixed
>
> > +ok($dump !=~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped
> children table");
>
> !=~ is being interpretted as as numeric "!=" and throwing warnings.
> It should be a !~ b, right ?
> It'd be nice if perl warnings during the tests were less easy to miss.
>
should be fixed by you
>
> > + * char is not alpha. The char '_' is allowed too (exclude first
> position).
>
>
> Why is it treated specially? Could it be treated the same as alpha?
>
It is usual behaviour in Postgres for keywords. Important is the complete
sentence "Returns NULL when the buffer is empty or the first char is not
alpha."
In this case this implementation has no big impact on behaviour - probably
you got a message "unknown keyword" instead of "missing keyword". But I
would
implement behaviour consistent with other places. My opinion in this case
is not extra strong - we can define the form of keywords like we want, just
this is consistent
with other parsers in Postgres.
>
> > + log_invalid_filter_format(&fstate,
> > +
> "\"include\" table data filter is not allowed");
> > + log_invalid_filter_format(&fstate,
> > +
> "\"include\" table data and children filter is not allowed");
>
> For these, it might be better to write the literal option:
>
> > +
> "include filter for \"table_data_and_children\" is not allowed");
>
> Because the option is a literal and shouldn't be translated.
> And it's probably better to write that using %s, like:
>
> > +
> "include filter for \"%s\" is not allowed");
>
done
>
> That makes shorter and fewer strings.
>
> Find attached a bunch of other corrections as 0002.txt
>
merged
Regards
Pavel
Attachments:
[text/x-patch] v20230320-0001-possibility-to-read-options-for-dump-from-file.patch (60.4K, ../../CAFj8pRDN8_jqzk8SppfYLeXw02XeNf_zYO28ZjjPf1hcQFe30g@mail.gmail.com/3-v20230320-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 844b54e0dcdb65dbf7f86a75b977db16dad07254 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 107 +++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 509 ++++++++++++++
src/bin/pg_dump/filter.h | 57 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 114 ++++
src/bin/pg_dump/pg_dumpall.c | 60 +-
src/bin/pg_dump/pg_restore.c | 110 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 701 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1710 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d6b1faa804..f3e287b75a 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -829,6 +829,99 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { table | table_and_children | schema | foreign_data | table_data | table_data_and_children } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-childrent</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1159,6 +1252,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1581,6 +1675,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..547fe3803f 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index eb8f59459a..bff55b6b1a 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -30,6 +30,7 @@ OBJS = \
compress_lz4.o \
compress_none.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -47,8 +48,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..ef77459038
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,509 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+static void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("\"%s\" doesn't support filter for object type \"%s\"."),
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * Emit error message "exclude" or "include" filter for filter type
+ * is not allowed.
+ */
+void
+log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot,
+ bool is_include)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("%s filter for \"%s\" is not allowed."),
+ is_include ? "include" : "exclude",
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, _("unexpected end of file"));
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, _("missing object name pattern"));
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude"
+ * object_type can one of: "table", "table_and_children", "schema", "foreign_data",
+ * "table_data", "table_and_children", "database", "function", "trigger" or "index"
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, _("missing filter object type"));
+ return false;
+ }
+
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, _("unsupported filter object type: \"%.*s\""), size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..9da8fd4afa
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,57 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern void log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot, bool is_include);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index b2fb7ac77f..0a626e6cc6 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -6,6 +6,7 @@ pg_dump_common_sources = files(
'compress_lz4.c',
'compress_none.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -97,6 +98,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index d62780a088..5aaa5b0b35 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -326,6 +327,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -404,6 +406,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -656,6 +659,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 15: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1102,6 +1109,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18475,3 +18484,108 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children, objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index cd421c5944..c060d96236 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1908,7 +1916,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1932,3 +1939,54 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..71414b27a1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,105 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..5cb26919b8
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,701 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 96;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index e3ffc653e5..fca7b7d0de 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -451,6 +451,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.40.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-03-21 15:32 ` Justin Pryzby <[email protected]>
2023-03-21 15:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 2 replies; 19+ messages in thread
From: Justin Pryzby @ 2023-03-21 15:32 UTC (permalink / raw)
To: Pavel Stehule <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
On Mon, Mar 20, 2023 at 08:01:13AM +0100, Pavel Stehule wrote:
> ne 19. 3. 2023 v 15:01 odesílatel Justin Pryzby <[email protected]> napsal:
>
> > On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> > > rebase + enhancing about related option from a563c24
> >
> > Thanks.
> >
> > It looks like this doesn't currently handle extensions, which were added
> > at 6568cef26e.
What about this part ? Should extension filters be supported ?
I think the comment that I'd patched that lists all the filter types
should be minimized, rather than duplicating the list of all the
possible filters that's already in the usrr-facing documentation.
One new typo: childrent
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
@ 2023-03-21 15:33 ` Pavel Stehule <[email protected]>
1 sibling, 0 replies; 19+ messages in thread
From: Pavel Stehule @ 2023-03-21 15:33 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
út 21. 3. 2023 v 16:32 odesílatel Justin Pryzby <[email protected]>
napsal:
> On Mon, Mar 20, 2023 at 08:01:13AM +0100, Pavel Stehule wrote:
> > ne 19. 3. 2023 v 15:01 odesílatel Justin Pryzby <[email protected]>
> napsal:
> >
> > > On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> > > > rebase + enhancing about related option from a563c24
> > >
> > > Thanks.
> > >
> > > It looks like this doesn't currently handle extensions, which were
> added
> > > at 6568cef26e.
>
> What about this part ? Should extension filters be supported ?
>
I missed this, yes, it should be supported.
>
> I think the comment that I'd patched that lists all the filter types
> should be minimized, rather than duplicating the list of all the
> possible filters that's already in the usrr-facing documentation.
>
> One new typo: childrent
>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
@ 2023-03-21 22:00 ` Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-03-21 22:00 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
út 21. 3. 2023 v 16:32 odesílatel Justin Pryzby <[email protected]>
napsal:
> On Mon, Mar 20, 2023 at 08:01:13AM +0100, Pavel Stehule wrote:
> > ne 19. 3. 2023 v 15:01 odesílatel Justin Pryzby <[email protected]>
> napsal:
> >
> > > On Thu, Mar 16, 2023 at 01:05:41PM +0100, Pavel Stehule wrote:
> > > > rebase + enhancing about related option from a563c24
> > >
> > > Thanks.
> > >
> > > It looks like this doesn't currently handle extensions, which were
> added
> > > at 6568cef26e.
>
> What about this part ? Should extension filters be supported ?
>
should be fixed
>
> I think the comment that I'd patched that lists all the filter types
> should be minimized, rather than duplicating the list of all the
> possible filters that's already in the usrr-facing documentation.
>
I modified this comment. Please, check
>
> One new typo: childrent
>
fixed
Regards
Pavel
Attachments:
[text/x-patch] v20230321-0001-possibility-to-read-options-for-dump-from-file.patch (62.0K, ../../CAFj8pRBqpMvqTLftpte5cQWa=8-Hd6nSZ6PiyXgRvRXu1-N_Ow@mail.gmail.com/3-v20230321-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From f91f525f32f51f7c5784dd7af57fe2b692db5e7f Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 530 +++++++++++++++
src/bin/pg_dump/filter.h | 58 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 127 ++++
src/bin/pg_dump/pg_dumpall.c | 60 +-
src/bin/pg_dump/pg_restore.c | 110 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1768 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d6b1faa804..17bfc661a9 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -829,6 +829,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1159,6 +1259,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1581,6 +1682,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..547fe3803f 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index eb8f59459a..bff55b6b1a 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -30,6 +30,7 @@ OBJS = \
compress_lz4.o \
compress_none.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -47,8 +48,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..be50fbf503
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,530 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+static void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("\"%s\" doesn't support filter for object type \"%s\"."),
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * Emit error message "exclude" or "include" filter for filter type
+ * is not allowed.
+ */
+void
+log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot,
+ bool is_include)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("%s filter for \"%s\" is not allowed."),
+ is_include ? "include" : "exclude",
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, _("unexpected end of file"));
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, _("missing object name pattern"));
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, _("missing filter object type"));
+ return false;
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, _("unsupported filter object type: \"%.*s\""), size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..9750e6a0d8
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern void log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot, bool is_include);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index b2fb7ac77f..0a626e6cc6 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -6,6 +6,7 @@ pg_dump_common_sources = files(
'compress_lz4.c',
'compress_none.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -97,6 +98,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index d62780a088..2b09418ad5 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -59,6 +59,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -326,6 +327,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -404,6 +406,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -656,6 +659,10 @@ main(int argc, char **argv)
optarg);
break;
+ case 15: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1102,6 +1109,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18475,3 +18484,121 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_EXTENSION)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&extension_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ }
+ else
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index cd421c5944..c060d96236 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1908,7 +1916,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1932,3 +1939,54 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..71414b27a1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,105 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..4ca63ca80b
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index e3ffc653e5..fca7b7d0de 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -451,6 +451,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.40.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-09-11 04:34 ` Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-09-11 04:34 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
only rebase
Regards
Pavel
Attachments:
[text/x-patch] v20230911-0001-possibility-to-read-options-for-dump-from-file.patch (61.9K, ../../CAFj8pRA25Kg1_skQKic1cyLThrNuwY+uDU4ozuCsm2JthmdLdw@mail.gmail.com/3-v20230911-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From b62d99f51da03b1fb8dae577fc49420bf36a3bad Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 530 +++++++++++++++
src/bin/pg_dump/filter.h | 58 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 126 ++++
src/bin/pg_dump/pg_dumpall.c | 60 +-
src/bin/pg_dump/pg_restore.c | 110 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1767 insertions(+), 3 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index c1e2220b3c..ae1cc522a8 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -835,6 +835,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1165,6 +1265,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1608,6 +1709,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..547fe3803f 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 24de7593a6..14765fc8b1 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..be50fbf503
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,530 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+static void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("\"%s\" doesn't support filter for object type \"%s\"."),
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * Emit error message "exclude" or "include" filter for filter type
+ * is not allowed.
+ */
+void
+log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot,
+ bool is_include)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("%s filter for \"%s\" is not allowed."),
+ is_include ? "include" : "exclude",
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, _("unexpected end of file"));
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, _("missing object name pattern"));
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, _("missing filter object type"));
+ return false;
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, _("unsupported filter object type: \"%.*s\""), size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..9750e6a0d8
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern void log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot, bool is_include);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f7b6176692..9082357933 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -406,6 +408,7 @@ main(int argc, char **argv)
{"enable-row-security", no_argument, &dopt.enable_row_security, 1},
{"exclude-table-data", required_argument, NULL, 4},
{"extra-float-digits", required_argument, NULL, 8},
+ {"filter", required_argument, NULL, 15},
{"if-exists", no_argument, &dopt.if_exists, 1},
{"inserts", no_argument, NULL, 9},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -662,6 +665,9 @@ main(int argc, char **argv)
case 15:
if (!parse_sync_method(optarg, &sync_method))
exit_nicely(1);
+
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
break;
default:
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18752,3 +18760,121 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_EXTENSION)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&extension_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ }
+ else
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..17cf182455 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,54 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..71414b27a1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,105 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..4ca63ca80b
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 9e05eb91b1..1d92cc676d 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -453,6 +453,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.41.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-09-11 04:57 ` Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-09-11 04:57 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
po 11. 9. 2023 v 6:34 odesílatel Pavel Stehule <[email protected]>
napsal:
> Hi
>
> only rebase
>
Unfortunately this rebase was not correct. I am sorry.
fixed version
Regards
Pavel
>
> Regards
>
> Pavel
>
>
Attachments:
[text/x-patch] v20230911-2-0001-possibility-to-read-options-for-dump-from-file.patch (61.8K, ../../CAFj8pRAHQb0CBh33rrJUc67f+2Meug8NqWa1Qfv8CaDCzQ+CCQ@mail.gmail.com/3-v20230911-2-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 32ea3d180ccd976b266bb48c5445c96a1aaf7a54 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 530 +++++++++++++++
src/bin/pg_dump/filter.h | 58 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 127 +++-
src/bin/pg_dump/pg_dumpall.c | 60 +-
src/bin/pg_dump/pg_restore.c | 110 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1767 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index c1e2220b3c..ae1cc522a8 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -835,6 +835,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1165,6 +1265,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1608,6 +1709,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..547fe3803f 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 24de7593a6..14765fc8b1 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..be50fbf503
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,530 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+static void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("\"%s\" doesn't support filter for object type \"%s\"."),
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * Emit error message "exclude" or "include" filter for filter type
+ * is not allowed.
+ */
+void
+log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot,
+ bool is_include)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("%s filter for \"%s\" is not allowed."),
+ is_include ? "include" : "exclude",
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, _("unexpected end of file"));
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, _("missing object name pattern"));
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, _("missing filter object type"));
+ return false;
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, _("unsupported filter object type: \"%.*s\""), size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..9750e6a0d8
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern void log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot, bool is_include);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f7b6176692..9f652e4b8b 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -662,6 +664,9 @@ main(int argc, char **argv)
case 15:
if (!parse_sync_method(optarg, &sync_method))
exit_nicely(1);
+
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
break;
default:
@@ -1111,6 +1116,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18752,3 +18759,121 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_EXTENSION)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&extension_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ }
+ else
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..17cf182455 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,54 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..71414b27a1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,105 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..4ca63ca80b
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 9e05eb91b1..1d92cc676d 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -453,6 +453,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.41.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-09-11 06:33 ` Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-09-11 06:33 UTC (permalink / raw)
To: Justin Pryzby <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Daniel Gustafsson <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
po 11. 9. 2023 v 6:57 odesílatel Pavel Stehule <[email protected]>
napsal:
> Hi
>
> po 11. 9. 2023 v 6:34 odesílatel Pavel Stehule <[email protected]>
> napsal:
>
>> Hi
>>
>> only rebase
>>
>
> Unfortunately this rebase was not correct. I am sorry.
>
> fixed version
>
and fixed forgotten "break" in switch
Regards
Pavel
>
> Regards
>
> Pavel
>
>
>>
>> Regards
>>
>> Pavel
>>
>>
Attachments:
[text/x-patch] v20230911-2-0001-possibility-to-read-options-for-dump-from-file.patch (61.9K, ../../CAFj8pRAKFoPSg392t_Mc1612H7icdf2cKXt_vtzObtyK=PwaNw@mail.gmail.com/3-v20230911-2-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 1f0738992d69d0d748bd4494a9244c353c224ace Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 530 +++++++++++++++
src/bin/pg_dump/filter.h | 58 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 128 +++-
src/bin/pg_dump/pg_dumpall.c | 60 +-
src/bin/pg_dump/pg_restore.c | 110 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1768 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index c1e2220b3c..ae1cc522a8 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -835,6 +835,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1165,6 +1265,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1608,6 +1709,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index e219a79858..547fe3803f 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -122,6 +122,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 47bd7dbda0..ffeb564c52 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -188,6 +188,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 24de7593a6..14765fc8b1 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..be50fbf503
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,530 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+bool
+filter_init(FilterStateData *fstate, const char *filename)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ return false;
+ }
+ }
+ else
+ fstate->fp = stdin;
+
+ fstate->is_error = false;
+
+ return true;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+static const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data",keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+/*
+ * Emit error message "invalid format in filter file ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+static void
+log_invalid_filter_format(FilterStateData *fstate, char *message)
+{
+ if (fstate->fp != stdin)
+ {
+ pg_log_error("invalid format in filter file \"%s\" on line %d: %s",
+ fstate->filename,
+ fstate->lineno,
+ message);
+ }
+ else
+ pg_log_error("invalid format in filter on line %d: %s",
+ fstate->lineno,
+ message);
+
+ fstate->is_error = true;
+}
+
+/*
+ * Emit error message "The application doesn't support filter for object type ..."
+ *
+ * This is mostly a convenience routine to avoid duplicating file closing code
+ * in multiple callsites.
+ */
+void
+log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname,
+ FilterObjectType fot)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("\"%s\" doesn't support filter for object type \"%s\"."),
+ appname,
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * Emit error message "exclude" or "include" filter for filter type
+ * is not allowed.
+ */
+void
+log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot,
+ bool is_include)
+{
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str,
+ _("%s filter for \"%s\" is not allowed."),
+ is_include ? "include" : "exclude",
+ filter_object_type_name(fot));
+
+ log_invalid_filter_format(fstate, str->data);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi lined string.
+ *
+ * Returns pointer to next char after ending double quotes or NULL on error.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ fstate->is_error = true;
+ }
+ else
+ log_invalid_filter_format(fstate, _("unexpected end of file"));
+
+ return NULL;
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier, or NULL on
+ * error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ log_invalid_filter_format(fstate, _("missing object name pattern"));
+ return NULL;
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found
+ * in original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ if (!str)
+ return NULL;
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ Assert(!fstate->is_error);
+
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ log_invalid_filter_format(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ return false;
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ log_invalid_filter_format(fstate, _("missing filter object type"));
+ return false;
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ PQExpBuffer str = createPQExpBuffer();
+
+ printfPQExpBuffer(str, _("unsupported filter object type: \"%.*s\""), size, keyword);
+ log_invalid_filter_format(fstate, str->data);
+ return false;
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ if (!str)
+ return false;
+
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->is_error = true;
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..9750e6a0d8
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ int lineno;
+ StringInfoData linebuff;
+ bool is_error;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern bool filter_init(FilterStateData *fstate, const char *filename);
+extern void filter_free(FilterStateData *fstate);
+extern void log_unsupported_filter_object_type(FilterStateData *fstate,
+ const char *appname, FilterObjectType fot);
+extern void log_unallowed_filter_type(FilterStateData *fstate,
+ FilterObjectType fot, bool is_include);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index f7b6176692..33d90830b8 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -664,6 +666,10 @@ main(int argc, char **argv)
exit_nicely(1);
break;
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18752,3 +18760,121 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments and empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ else
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_EXTENSION)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&extension_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ }
+ else
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_FOREIGN_DATA)
+ {
+ if (is_include)
+ simple_string_list_append(&foreign_servers_include_patterns,
+ objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&schema_include_patterns,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&schema_exclude_patterns,
+ objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN)
+ {
+ if (is_include)
+ {
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ }
+ else
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dump", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..17cf182455 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,54 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_DATABASE)
+ {
+ if (!is_include)
+ simple_string_list_append(pattern, objname);
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_dumpall", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..71414b27a1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,105 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ if (!filter_init(&fstate, filename))
+ exit_nicely(1);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ /* ignore comments or empty lines */
+ if (objtype == FILTER_OBJECT_TYPE_NONE)
+ continue;
+
+ if (objtype == FILTER_OBJECT_TYPE_FUNCTION)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_INDEX)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_SCHEMA)
+ {
+ if (is_include)
+ simple_string_list_append(&opts->schemaNames, objname);
+ else
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TABLE)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else if (objtype == FILTER_OBJECT_TYPE_TRIGGER)
+ {
+ if (is_include)
+ {
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ }
+ else
+ {
+ log_unallowed_filter_type(&fstate, objtype, is_include);
+ break;
+ }
+ }
+ else
+ {
+ log_unsupported_filter_object_type(&fstate, "pg_restore", objtype);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+ if (fstate.is_error)
+ exit_nicely(1);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..4ca63ca80b
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/"pg_dumpall" doesn't support filter for object type "table"/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 9e05eb91b1..1d92cc676d 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -453,6 +453,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.41.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-11-09 14:26 ` Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Daniel Gustafsson @ 2023-11-09 14:26 UTC (permalink / raw)
To: Pavel Stehule <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
I went and had another look at this. The patch has been around for 18
commitfests and is widely considered to add a good feature, so it seems about
time to get reach closure.
As I've mentioned in the past I'm not a big fan of the parser, but the thread
has overruled on that. Another thing I think is a bit overcomplicated is the
layered error handling for printing log messages, and bubbling up of errors to
get around not being able to call exit_nicely.
In the attached version I've boiled down the error logging into a single new
function pg_log_filter_error() which takes a variable format string. This
removes a fair bit of the extra calls and makes logging easier. I've also
added a function pointer to the FilterStateData for passing the exit function
via filter_init. This allows the filtering code to exit gracefully regardless
of which application is using it. Finally, I've also reimplemented the logic
for checking the parsed tokens into switch statements without defaults in order
to get the compilerwarning on a missed case. It's easy to miss adding code to
handle a state, especially when adding new ones, and this should help highlight
that.
Overall, this does shave a bit off the patch in size for what IMHO is better
readability and maintainability. (I've also made a pgindent pass over it of
course).
What are your thoughts on this version? It's not in a committable state as it
needs a bit more comments here and there and a triplecheck that nothing was
missed in changing this, but I prefer to get your thoughts before spending the
extra time.
--
Daniel Gustafsson
Attachments:
[application/octet-stream] v20231109-0001-possibility-to-read-options-for-dump-from-.patch (61.7K, ../../[email protected]/2-v20231109-0001-possibility-to-read-options-for-dump-from-.patch)
download | inline diff:
From faf49822f377b28f9fd10fa8aa88a4e0749c141a Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Thu, 16 Mar 2023 08:18:08 +0100
Subject: [PATCH v20231109] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 471 +++++++++++++
src/bin/pg_dump/filter.h | 59 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 114 +++-
src/bin/pg_dump/pg_dumpall.c | 68 +-
src/bin/pg_dump/pg_restore.c | 103 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1697 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8695571045..e2f100d552 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -836,6 +836,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1168,6 +1268,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1611,6 +1712,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index d31585216c..75ba03f3ad 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -125,6 +125,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 374d8d8715..64f7c5dc4d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -190,6 +190,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 24de7593a6..14765fc8b1 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..46970a57e3
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+void
+filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ fstate->exit_nicely = f_exit;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ fstate->exit_nicely(1);
+ }
+ }
+ else
+ fstate->fp = stdin;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ if (!fstate)
+ return;
+
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+void
+pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+{
+ va_list argp;
+ char buf[256];
+
+ va_start(argp, fmt);
+ vsnprintf(buf, sizeof(buf), fmt, argp);
+ va_end(argp);
+
+ pg_log_error("invalid format in filter \"%s\" on line %d: %s",
+ (fstate->fp == stdin ? "stdin" : fstate->filename),
+ fstate->lineno,
+ buf);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi line string
+ *
+ * Reads a quoted string which can span over multiple lines and returns a
+ * pointer to next char after ending double quotes; it will exit on errors.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ else
+ pg_log_filter_error(fstate, _("unexpected end of file"));
+
+ fstate->exit_nicely(1);
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier and exits
+ * on error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ pg_log_filter_error(fstate, _("missing object name pattern"));
+ fstate->exit_nicely(1);
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found in
+ * original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ pg_log_filter_error(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate, _("missing filter object type"));
+ fstate->exit_nicely(1);
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ pg_log_filter_error(fstate,
+ _("unsupported filter object type: \"%.*s\""), size, keyword);
+ fstate->exit_nicely(1);
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->exit_nicely(1);
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..8e8fc6faee
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,59 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/* Function signature for exit_nicely functions */
+typedef void (*exit_function) (int status);
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ exit_function exit_nicely;
+ int lineno;
+ StringInfoData linebuff;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern const char *filter_object_type_name(FilterObjectType fot);
+extern void filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit);
+extern void filter_free(FilterStateData *fstate);
+extern void pg_log_filter_error(FilterStateData *fstate, const char *fmt,...);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e863913849..3a4ba7cfbe 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -664,6 +666,10 @@ main(int argc, char **argv)
exit_nicely(1);
break;
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18769,3 +18777,107 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break; /* unreachable */
+
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ simple_string_list_append(&extension_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ simple_string_list_append(&foreign_servers_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..6d21128818 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,62 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ }
+
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ pg_log_filter_error(&fstate, _("unsupported filter object."));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_DATABASE:
+ simple_string_list_append(pattern, objname);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..f647bde28d 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,98 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_INDEX:
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..a0aee12543
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/pg_dumpall: error: invalid format in filter/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 84f648c174..bcbcd8116f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -455,6 +455,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.32.1 (Apple Git-133)
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
@ 2023-11-12 13:17 ` Pavel Stehule <[email protected]>
2023-11-13 13:15 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-11-12 13:17 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
> What are your thoughts on this version? It's not in a committable state
> as it
> needs a bit more comments here and there and a triplecheck that nothing was
> missed in changing this, but I prefer to get your thoughts before spending
> the
> extra time.
>
I think using pointer to exit function is an elegant solution. I checked
the code and I found only one issue. I fixed warning
[13:57:22.578] time make -s -j${BUILD_JOBS} world-bin
[13:58:20.858] filter.c: In function ‘pg_log_filter_error’:
[13:58:20.858] filter.c:161:2: error: function ‘pg_log_filter_error’ might
be a candidate for ‘gnu_printf’ format attribute
[-Werror=suggest-attribute=format]
[13:58:20.858] 161 | vsnprintf(buf, sizeof(buf), fmt, argp);
[13:58:20.858] | ^~~~~~~~~
[13:58:20.858] cc1: all warnings being treated as errors
and probably copy/paste bug
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index f647bde28d..ab2abedf5f 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -535,7 +535,7 @@ read_restore_filters(const char *filename,
RestoreOptions *opts)
case FILTER_OBJECT_TYPE_EXTENSION:
case FILTER_OBJECT_TYPE_FOREIGN_DATA:
pg_log_filter_error(&fstate, _("%s filter for \"%s\" is
not allowed."),
- "exclude",
+ "include",
filter_object_type_name(objtype));
exit_nicely(1);
Regards
Pavel
>
> --
> Daniel Gustafsson
>
>
Attachments:
[text/x-patch] v20231112-0002-fix-err-message.patch (838B, ../../CAFj8pRA4LebAbgT-VyhJbrZzzVJ2d8p5K1vBrzJuxWBKeHa8iw@mail.gmail.com/3-v20231112-0002-fix-err-message.patch)
download | inline diff:
From 9be8fb14a3fe75aa4203a059e8372986bf5e6615 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Sun, 12 Nov 2023 13:54:26 +0100
Subject: [PATCH 2/2] fix err message
---
src/bin/pg_dump/pg_restore.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index f647bde28d..ab2abedf5f 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -535,7 +535,7 @@ read_restore_filters(const char *filename, RestoreOptions *opts)
case FILTER_OBJECT_TYPE_EXTENSION:
case FILTER_OBJECT_TYPE_FOREIGN_DATA:
pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
- "exclude",
+ "include",
filter_object_type_name(objtype));
exit_nicely(1);
--
2.41.0
[text/x-patch] v20231112-0001-possibility-to-read-options-for-dump-from-file.patch (61.7K, ../../CAFj8pRA4LebAbgT-VyhJbrZzzVJ2d8p5K1vBrzJuxWBKeHa8iw@mail.gmail.com/4-v20231112-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From cbaae854eca0cc88bb0886abfd45416ad13cffc7 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Sat, 11 Nov 2023 20:34:34 +0100
Subject: [PATCH 1/2] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 471 +++++++++++++
src/bin/pg_dump/filter.h | 60 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 114 +++-
src/bin/pg_dump/pg_dumpall.c | 68 +-
src/bin/pg_dump/pg_restore.c | 103 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1698 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8695571045..e2f100d552 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -836,6 +836,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1168,6 +1268,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1611,6 +1712,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index d31585216c..75ba03f3ad 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -125,6 +125,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 374d8d8715..64f7c5dc4d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -190,6 +190,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 604cddb997..2bcf2a7002 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..46970a57e3
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+void
+filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ fstate->exit_nicely = f_exit;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ fstate->exit_nicely(1);
+ }
+ }
+ else
+ fstate->fp = stdin;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ if (!fstate)
+ return;
+
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+void
+pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+{
+ va_list argp;
+ char buf[256];
+
+ va_start(argp, fmt);
+ vsnprintf(buf, sizeof(buf), fmt, argp);
+ va_end(argp);
+
+ pg_log_error("invalid format in filter \"%s\" on line %d: %s",
+ (fstate->fp == stdin ? "stdin" : fstate->filename),
+ fstate->lineno,
+ buf);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi line string
+ *
+ * Reads a quoted string which can span over multiple lines and returns a
+ * pointer to next char after ending double quotes; it will exit on errors.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ else
+ pg_log_filter_error(fstate, _("unexpected end of file"));
+
+ fstate->exit_nicely(1);
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier and exits
+ * on error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ pg_log_filter_error(fstate, _("missing object name pattern"));
+ fstate->exit_nicely(1);
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found in
+ * original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ pg_log_filter_error(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate, _("missing filter object type"));
+ fstate->exit_nicely(1);
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ pg_log_filter_error(fstate,
+ _("unsupported filter object type: \"%.*s\""), size, keyword);
+ fstate->exit_nicely(1);
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->exit_nicely(1);
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..cd5c2cf5f7
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,60 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/* Function signature for exit_nicely functions */
+typedef void (*exit_function) (int status);
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ exit_function exit_nicely;
+ int lineno;
+ StringInfoData linebuff;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern const char *filter_object_type_name(FilterObjectType fot);
+extern void filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit);
+extern void filter_free(FilterStateData *fstate);
+extern void pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+ pg_attribute_printf(2, 3);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e863913849..3a4ba7cfbe 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -664,6 +666,10 @@ main(int argc, char **argv)
exit_nicely(1);
break;
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18769,3 +18777,107 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break; /* unreachable */
+
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ simple_string_list_append(&extension_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ simple_string_list_append(&foreign_servers_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..6d21128818 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,62 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ }
+
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ pg_log_filter_error(&fstate, _("unsupported filter object."));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_DATABASE:
+ simple_string_list_append(pattern, objname);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..f647bde28d 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,98 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_INDEX:
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..a0aee12543
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/pg_dumpall: error: invalid format in filter/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 84f648c174..bcbcd8116f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -455,6 +455,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.41.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-11-13 13:15 ` Pavel Stehule <[email protected]>
2023-11-13 13:39 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-11-13 13:15 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
ne 12. 11. 2023 v 14:17 odesílatel Pavel Stehule <[email protected]>
napsal:
> Hi
>
>
>> What are your thoughts on this version? It's not in a committable state
>> as it
>> needs a bit more comments here and there and a triplecheck that nothing
>> was
>> missed in changing this, but I prefer to get your thoughts before
>> spending the
>> extra time.
>>
>
> I think using pointer to exit function is an elegant solution. I checked
> the code and I found only one issue. I fixed warning
>
> [13:57:22.578] time make -s -j${BUILD_JOBS} world-bin
> [13:58:20.858] filter.c: In function ‘pg_log_filter_error’:
> [13:58:20.858] filter.c:161:2: error: function ‘pg_log_filter_error’ might
> be a candidate for ‘gnu_printf’ format attribute
> [-Werror=suggest-attribute=format]
> [13:58:20.858] 161 | vsnprintf(buf, sizeof(buf), fmt, argp);
> [13:58:20.858] | ^~~~~~~~~
> [13:58:20.858] cc1: all warnings being treated as errors
>
> and probably copy/paste bug
>
> diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
> index f647bde28d..ab2abedf5f 100644
> --- a/src/bin/pg_dump/pg_restore.c
> +++ b/src/bin/pg_dump/pg_restore.c
> @@ -535,7 +535,7 @@ read_restore_filters(const char *filename,
> RestoreOptions *opts)
> case FILTER_OBJECT_TYPE_EXTENSION:
> case FILTER_OBJECT_TYPE_FOREIGN_DATA:
> pg_log_filter_error(&fstate, _("%s filter for \"%s\"
> is not allowed."),
> - "exclude",
> + "include",
> filter_object_type_name(objtype));
> exit_nicely(1);
>
> Regards
>
> Pavel
>
next update - fix used, but uninitialized "is_include" variable, when
filter is of FILTER_OBJECT_TYPE_NONE
fix crash
# Running: pg_ctl -w -D
/tmp/cirrus-ci-build/build-32/testrun/pg_dump/005_pg_dump_filterfile/data/t_005_pg_dump_filterfile_main_data/pgdata
-l /tmp/cirrus-ci-build/build-32/testrun/pg_dump/005_pg_dump_filterfile/log/005_pg_dump_filterfile_main.log
-o --cluster-name=main start
waiting for server to start.... done
server started
# Postmaster PID for node "main" is 71352
# Running: pg_dump -p 65454 -f
/tmp/cirrus-ci-build/build-32/testrun/pg_dump/005_pg_dump_filterfile/data/t_005_pg_dump_filterfile_main_data/backup/plain.sql
--filter=/tmp/cirrus-ci-build/build-32/testrun/pg_dump/005_pg_dump_filterfile/data/tmp_test_0mO3/inputfile.txt
postgres
../src/bin/pg_dump/pg_dump.c:18800:7: runtime error: load of value 86,
which is not a valid value for type '_Bool'
==71579==Using libbacktrace symbolizer.
#0 0x566302cd in read_dump_filters ../src/bin/pg_dump/pg_dump.c:18800
#1 0x56663429 in main ../src/bin/pg_dump/pg_dump.c:670
#2 0xf7694e45 in __libc_start_main (/lib/i386-linux-gnu/libc.so.6+0x1ae45)
#3 0x56624d50 in _start
(/tmp/cirrus-ci-build/build-32/tmp_install/usr/local/pgsql/bin/pg_dump+0x1ad50)
Regards
Pavel
>
>
>>
>> --
>> Daniel Gustafsson
>>
>>
Attachments:
[text/x-patch] v20231113-0004-fix-uninitialize-is_include-variable-runtime-error-l.patch (710B, ../../CAFj8pRAZ8DNZY_Wevp7JvD+vw3B87E=B39qkKuXUG9tUCzyDAg@mail.gmail.com/3-v20231113-0004-fix-uninitialize-is_include-variable-runtime-error-l.patch)
download | inline diff:
From 235bcd7944e99468c3fe6a65ca5490883b983a70 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Mon, 13 Nov 2023 14:10:49 +0100
Subject: [PATCH 4/4] fix uninitialize is_include variable, runtime error: load
of value 86, which is not a valid value for type '_Bool'
---
src/bin/pg_dump/filter.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
index 46970a57e3..2d724b1b35 100644
--- a/src/bin/pg_dump/filter.c
+++ b/src/bin/pg_dump/filter.c
@@ -455,6 +455,7 @@ filter_read_item(FilterStateData *fstate,
else
{
*objname = NULL;
+ *is_include = false;
*objtype = FILTER_OBJECT_TYPE_NONE;
}
--
2.41.0
[text/x-patch] v20231113-0003-add-more-tests-related-to-unsupported-options-of-pg_.patch (2.6K, ../../CAFj8pRAZ8DNZY_Wevp7JvD+vw3B87E=B39qkKuXUG9tUCzyDAg@mail.gmail.com/4-v20231113-0003-add-more-tests-related-to-unsupported-options-of-pg_.patch)
download | inline diff:
From 13984bc6ca65b20c4a23a65602304ab0bf11c955 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Mon, 13 Nov 2023 08:26:22 +0100
Subject: [PATCH 3/4] add more tests related to unsupported options of
pg_restore
---
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 58 ++++++++++++++++++++-
1 file changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
index a0aee12543..09d3262b8b 100644
--- a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -6,7 +6,7 @@ use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
-use Test::More tests => 98;
+use Test::More tests => 106;
my $tempdir = PostgreSQL::Test::Utils::tempdir;;
my $inputfile;
@@ -535,6 +535,62 @@ $dump = slurp_file($plainfile);
ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table_data xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/include filter for "table data" is not allowed/,
+ "invalid syntax: inclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/include filter for "extension" is not allowed/,
+ "invalid syntax: inclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude extension xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/exclude filter for "extension" is not allowed/,
+ "invalid syntax: exclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table_data xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/xclude filter for "table data" is not allowed/,
+ "invalid syntax: exclusion of non allowed object"
+);
+
#########################################
# test restore of other objects
--
2.41.0
[text/x-patch] v20231113-0002-fix-err-message.patch (838B, ../../CAFj8pRAZ8DNZY_Wevp7JvD+vw3B87E=B39qkKuXUG9tUCzyDAg@mail.gmail.com/5-v20231113-0002-fix-err-message.patch)
download | inline diff:
From 0f9ca3ebcbfd27f78c30e873002081b1354d2d1f Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Sun, 12 Nov 2023 13:54:26 +0100
Subject: [PATCH 2/4] fix err message
---
src/bin/pg_dump/pg_restore.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index f647bde28d..ab2abedf5f 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -535,7 +535,7 @@ read_restore_filters(const char *filename, RestoreOptions *opts)
case FILTER_OBJECT_TYPE_EXTENSION:
case FILTER_OBJECT_TYPE_FOREIGN_DATA:
pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
- "exclude",
+ "include",
filter_object_type_name(objtype));
exit_nicely(1);
--
2.41.0
[text/x-patch] v20231113-0001-possibility-to-read-options-for-dump-from-file.patch (61.7K, ../../CAFj8pRAZ8DNZY_Wevp7JvD+vw3B87E=B39qkKuXUG9tUCzyDAg@mail.gmail.com/6-v20231113-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From a7a598b40948a5788f509c630f664fa16861e215 Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Sat, 11 Nov 2023 20:34:34 +0100
Subject: [PATCH 1/4] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 ++++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 471 +++++++++++++
src/bin/pg_dump/filter.h | 60 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 114 +++-
src/bin/pg_dump/pg_dumpall.c | 68 +-
src/bin/pg_dump/pg_restore.c | 103 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 717 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1698 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8695571045..e2f100d552 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -836,6 +836,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1168,6 +1268,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1611,6 +1712,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index d31585216c..75ba03f3ad 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -125,6 +125,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 374d8d8715..64f7c5dc4d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -190,6 +190,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 604cddb997..2bcf2a7002 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..46970a57e3
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+void
+filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ fstate->exit_nicely = f_exit;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ fstate->exit_nicely(1);
+ }
+ }
+ else
+ fstate->fp = stdin;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ if (!fstate)
+ return;
+
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+void
+pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+{
+ va_list argp;
+ char buf[256];
+
+ va_start(argp, fmt);
+ vsnprintf(buf, sizeof(buf), fmt, argp);
+ va_end(argp);
+
+ pg_log_error("invalid format in filter \"%s\" on line %d: %s",
+ (fstate->fp == stdin ? "stdin" : fstate->filename),
+ fstate->lineno,
+ buf);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi line string
+ *
+ * Reads a quoted string which can span over multiple lines and returns a
+ * pointer to next char after ending double quotes; it will exit on errors.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ else
+ pg_log_filter_error(fstate, _("unexpected end of file"));
+
+ fstate->exit_nicely(1);
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier and exits
+ * on error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ pg_log_filter_error(fstate, _("missing object name pattern"));
+ fstate->exit_nicely(1);
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found in
+ * original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ bool *is_include,
+ char **objname,
+ FilterObjectType *objtype)
+{
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *is_include = true;
+ else if (is_keyword_str("exclude", keyword, size))
+ *is_include = false;
+ else
+ {
+ pg_log_filter_error(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate, _("missing filter object type"));
+ fstate->exit_nicely(1);
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ pg_log_filter_error(fstate,
+ _("unsupported filter object type: \"%.*s\""), size, keyword);
+ fstate->exit_nicely(1);
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->exit_nicely(1);
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..cd5c2cf5f7
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,60 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/* Function signature for exit_nicely functions */
+typedef void (*exit_function) (int status);
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ exit_function exit_nicely;
+ int lineno;
+ StringInfoData linebuff;
+} FilterStateData;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER
+} FilterObjectType;
+
+extern const char *filter_object_type_name(FilterObjectType fot);
+extern void filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit);
+extern void filter_free(FilterStateData *fstate);
+extern void pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+ pg_attribute_printf(2, 3);
+extern bool filter_read_item(FilterStateData *fstate, bool *is_include,
+ char **objname, FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e863913849..3a4ba7cfbe 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -664,6 +666,10 @@ main(int argc, char **argv)
exit_nicely(1);
break;
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18769,3 +18777,107 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break; /* unreachable */
+
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ simple_string_list_append(&extension_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ simple_string_list_append(&foreign_servers_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..6d21128818 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,62 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ }
+
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ pg_log_filter_error(&fstate, _("unsupported filter object."));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_DATABASE:
+ simple_string_list_append(pattern, objname);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..f647bde28d 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,98 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ bool is_include;
+ char *objname;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &is_include, &objname, &objtype))
+ {
+ if (is_include)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_INDEX:
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ break;
+ }
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ break;
+ }
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..a0aee12543
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,717 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 98;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/pg_dumpall: error: invalid format in filter/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 84f648c174..bcbcd8116f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -455,6 +455,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.41.0
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:15 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-11-13 13:39 ` Daniel Gustafsson <[email protected]>
2023-11-13 16:07 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Daniel Gustafsson @ 2023-11-13 13:39 UTC (permalink / raw)
To: Pavel Stehule <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
> On 13 Nov 2023, at 14:15, Pavel Stehule <[email protected]> wrote:
>
> Hi
>
> ne 12. 11. 2023 v 14:17 odesílatel Pavel Stehule <[email protected] <mailto:[email protected]>> napsal:
> Hi
>
>
> What are your thoughts on this version? It's not in a committable state as it
> needs a bit more comments here and there and a triplecheck that nothing was
> missed in changing this, but I prefer to get your thoughts before spending the
> extra time.
>
> I think using pointer to exit function is an elegant solution. I checked the code and I found only one issue. I fixed warning
>
> [13:57:22.578] time make -s -j${BUILD_JOBS} world-bin
> [13:58:20.858] filter.c: In function ‘pg_log_filter_error’:
> [13:58:20.858] filter.c:161:2: error: function ‘pg_log_filter_error’ might be a candidate for ‘gnu_printf’ format attribute [-Werror=suggest-attribute=format]
> [13:58:20.858] 161 | vsnprintf(buf, sizeof(buf), fmt, argp);
> [13:58:20.858] | ^~~~~~~~~
> [13:58:20.858] cc1: all warnings being treated as errors
>
> and probably copy/paste bug
>
> diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
> index f647bde28d..ab2abedf5f 100644
> --- a/src/bin/pg_dump/pg_restore.c
> +++ b/src/bin/pg_dump/pg_restore.c
> @@ -535,7 +535,7 @@ read_restore_filters(const char *filename, RestoreOptions *opts)
> case FILTER_OBJECT_TYPE_EXTENSION:
> case FILTER_OBJECT_TYPE_FOREIGN_DATA:
> pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
> - "exclude",
> + "include",
> filter_object_type_name(objtype));
> exit_nicely(1);
>
> Regards
>
> Pavel
>
> next update - fix used, but uninitialized "is_include" variable, when filter is of FILTER_OBJECT_TYPE_NONE
Thanks, the posted patchset was indeed a bit of a sketch, thanks for fixing up
these. I'll go over it again too to clean it up and try to make into something
committable.
I was pondering replacing the is_include handling with returning an enum for
the operation, to keep things more future proof in case we add more operations
(and also a bit less magic IMHO).
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:15 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:39 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
@ 2023-11-13 16:07 ` Pavel Stehule <[email protected]>
2023-11-20 05:20 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Pavel Stehule @ 2023-11-13 16:07 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
po 13. 11. 2023 v 14:39 odesílatel Daniel Gustafsson <[email protected]>
napsal:
> > On 13 Nov 2023, at 14:15, Pavel Stehule <[email protected]> wrote:
> >
> > Hi
> >
> > ne 12. 11. 2023 v 14:17 odesílatel Pavel Stehule <
> [email protected] <mailto:[email protected]>> napsal:
> > Hi
> >
> >
> > What are your thoughts on this version? It's not in a committable state
> as it
> > needs a bit more comments here and there and a triplecheck that nothing
> was
> > missed in changing this, but I prefer to get your thoughts before
> spending the
> > extra time.
> >
> > I think using pointer to exit function is an elegant solution. I checked
> the code and I found only one issue. I fixed warning
> >
> > [13:57:22.578] time make -s -j${BUILD_JOBS} world-bin
> > [13:58:20.858] filter.c: In function ‘pg_log_filter_error’:
> > [13:58:20.858] filter.c:161:2: error: function ‘pg_log_filter_error’
> might be a candidate for ‘gnu_printf’ format attribute
> [-Werror=suggest-attribute=format]
> > [13:58:20.858] 161 | vsnprintf(buf, sizeof(buf), fmt, argp);
> > [13:58:20.858] | ^~~~~~~~~
> > [13:58:20.858] cc1: all warnings being treated as errors
> >
> > and probably copy/paste bug
> >
> > diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
> > index f647bde28d..ab2abedf5f 100644
> > --- a/src/bin/pg_dump/pg_restore.c
> > +++ b/src/bin/pg_dump/pg_restore.c
> > @@ -535,7 +535,7 @@ read_restore_filters(const char *filename,
> RestoreOptions *opts)
> > case FILTER_OBJECT_TYPE_EXTENSION:
> > case FILTER_OBJECT_TYPE_FOREIGN_DATA:
> > pg_log_filter_error(&fstate, _("%s filter for \"%s\"
> is not allowed."),
> > - "exclude",
> > + "include",
> >
> filter_object_type_name(objtype));
> > exit_nicely(1);
> >
> > Regards
> >
> > Pavel
> >
> > next update - fix used, but uninitialized "is_include" variable, when
> filter is of FILTER_OBJECT_TYPE_NONE
>
> Thanks, the posted patchset was indeed a bit of a sketch, thanks for
> fixing up
> these. I'll go over it again too to clean it up and try to make into
> something
> committable.
>
> I was pondering replacing the is_include handling with returning an enum
> for
> the operation, to keep things more future proof in case we add more
> operations
> (and also a bit less magic IMHO).
>
+1
Pavel
> --
> Daniel Gustafsson
>
>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: proposal: possibility to read dumped table's name from file
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:15 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:39 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-13 16:07 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
@ 2023-11-20 05:20 ` Pavel Stehule <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Pavel Stehule @ 2023-11-20 05:20 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Justin Pryzby <[email protected]>; Julien Rouhaud <[email protected]>; Gregory Stark (as CFM) <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; John Naylor <[email protected]>; Erik Rijkers <[email protected]>; Tom Lane <[email protected]>; Alvaro Herrera <[email protected]>; Tomas Vondra <[email protected]>; Dean Rasheed <[email protected]>; Stephen Frost <[email protected]>; Surafel Temesgen <[email protected]>; vignesh C <[email protected]>; pgsql-hackers
Hi
I was pondering replacing the is_include handling with returning an enum for
>> the operation, to keep things more future proof in case we add more
>> operations
>> (and also a bit less magic IMHO).
>>
>
> +1
>
I did it.
Regards
Pavel
>
> Pavel
>
>
>> --
>> Daniel Gustafsson
>>
>>
Attachments:
[text/x-patch] v20231120-0001-possibility-to-read-options-for-dump-from-file.patch (64.0K, ../../CAFj8pRD_yAue_M2WPMAU1jw2_ONyAGaTwsD1CyFTrwmFYo6Zvw@mail.gmail.com/3-v20231120-0001-possibility-to-read-options-for-dump-from-file.patch)
download | inline diff:
From 24631830e190cce814ee60a8050126ebc29b7ffa Mon Sep 17 00:00:00 2001
From: "[email protected]" <[email protected]>
Date: Sat, 11 Nov 2023 20:34:34 +0100
Subject: [PATCH] possibility to read options for dump from file
---
doc/src/sgml/ref/pg_dump.sgml | 114 +++
doc/src/sgml/ref/pg_dumpall.sgml | 22 +
doc/src/sgml/ref/pg_restore.sgml | 25 +
src/bin/pg_dump/Makefile | 5 +-
src/bin/pg_dump/filter.c | 472 ++++++++++++
src/bin/pg_dump/filter.h | 70 ++
src/bin/pg_dump/meson.build | 2 +
src/bin/pg_dump/pg_dump.c | 119 ++-
src/bin/pg_dump/pg_dumpall.c | 68 +-
src/bin/pg_dump/pg_restore.c | 108 +++
src/bin/pg_dump/t/005_pg_dump_filterfile.pl | 773 ++++++++++++++++++++
src/tools/msvc/Mkvcbuild.pm | 1 +
12 files changed, 1775 insertions(+), 4 deletions(-)
create mode 100644 src/bin/pg_dump/filter.c
create mode 100644 src/bin/pg_dump/filter.h
create mode 100644 src/bin/pg_dump/t/005_pg_dump_filterfile.pl
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8695571045..e2f100d552 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -836,6 +836,106 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects to include
+ or exclude from the dump. The patterns are interpreted according to the
+ same rules as the corresponding options:
+ <option>-t</option>/<option>--table</option>,
+ <option>--table-and-children</option>,
+ <option>--exclude-table-and-children</option> or
+ <option>-T</option> for tables,
+ <option>-n</option>/<option>--schema</option> for schemas,
+ <option>--include-foreign-data</option> for data on foreign servers and
+ <option>--exclude-table-data</option>,
+ <option>--exclude-table-data-and-children</option> for table data,
+ <option>-e</option>/<option>--extension</option> for extensions.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one object pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+
+ <para>
+ The first keyword specifies whether the objects matched by the pattern
+ are to be included or excluded. The second keyword specifies the type
+ of object to be filtered using the pattern:
+ <itemizedlist>
+ <listitem>
+ <para>
+ <literal>extension</literal>: data on foreign servers, works like
+ <option>--extension</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>foreign_data</literal>: data on foreign servers, works like
+ <option>--include-foreign-data</option>. This keyword can only be
+ used with the <literal>include</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table</literal>: tables, works like
+ <option>-t</option>/<option>--table</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_and_children</literal>: tables, works like
+ <option>--table-and-children</option>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data</literal>: table data, works like
+ <option>--exclude-table-data</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>table_data_and_children</literal>: table data of any
+ partitions or inheritance child, works like
+ <option>--exclude-table-data-and-children</option>. This keyword can only be
+ used with the <literal>exclude</literal> keyword.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ <literal>schema</literal>: schemas, works like
+ <option>-n</option>/<option>--schema</option>
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Lines starting with <literal>#</literal> are considered comments and
+ ignored. Comments can be placed after filter as well. Blank lines
+ are also ignored. See <xref linkend="app-psql-patterns"/> for how to
+ perform quoting in patterns.
+ </para>
+
+ <para>
+ Example files are listed below in the <xref linkend="pg-dump-examples"/>
+ section.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--if-exists</option></term>
<listitem>
@@ -1168,6 +1268,7 @@ PostgreSQL documentation
schema (<option>-n</option>/<option>--schema</option>) and
table (<option>-t</option>/<option>--table</option>) pattern
match at least one extension/schema/table in the database to be dumped.
+ This also applies to filters used with <option>--filter</option>.
Note that if none of the extension/schema/table patterns find
matches, <application>pg_dump</application> will generate an error
even without <option>--strict-names</option>.
@@ -1611,6 +1712,19 @@ CREATE DATABASE foo WITH TEMPLATE template0;
<screen>
<prompt>$</prompt> <userinput>pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql</userinput>
+</screen></para>
+
+ <para>
+ To dump all tables with names starting with mytable, except for table
+ <literal>mytable2</literal>, specify a filter file
+ <filename>filter.txt</filename> like:
+<programlisting>
+include table mytable*
+exclude table mytable2
+</programlisting>
+
+<screen>
+<prompt>$</prompt> <userinput>pg_dump --filter=filter.txt mydb > db.sql</userinput>
</screen></para>
</refsect1>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index d31585216c..75ba03f3ad 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -125,6 +125,28 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for databases excluded
+ from the dump. The patterns are interpretted according to the same rules
+ as <option>--exclude-database</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for excluding databases,
+ and can also be specified more than once for multiple filter files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+exclude database <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-g</option></term>
<term><option>--globals-only</option></term>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index 374d8d8715..64f7c5dc4d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -190,6 +190,31 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--filter=<replaceable class="parameter">filename</replaceable></option></term>
+ <listitem>
+ <para>
+ Specify a filename from which to read patterns for objects excluded
+ or included from restore. The patterns are interpretted according to the
+ same rules as <option>--schema</option>, <option>--exclude-schema</option>,
+ <option>--function</option>, <option>--index</option>, <option>--table</option>
+ or <option>--trigger</option>.
+ To read from <literal>STDIN</literal>, use <filename>-</filename> as the
+ filename. The <option>--filter</option> option can be specified in
+ conjunction with the above listed options for including or excluding
+ objects, and can also be specified more than once for multiple filter
+ files.
+ </para>
+
+ <para>
+ The file lists one database pattern per row, with the following format:
+<synopsis>
+{ include | exclude } { function | index | schema | table | trigger } <replaceable class="parameter">PATTERN</replaceable>
+</synopsis>
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-F <replaceable class="parameter">format</replaceable></option></term>
<term><option>--format=<replaceable class="parameter">format</replaceable></option></term>
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 604cddb997..2bcf2a7002 100644
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -32,6 +32,7 @@ OBJS = \
compress_none.o \
compress_zstd.o \
dumputils.o \
+ filter.o \
parallel.o \
pg_backup_archiver.o \
pg_backup_custom.o \
@@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg
pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
$(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
- $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
install: all installdirs
$(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X)
diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c
new file mode 100644
index 0000000000..9b5e45c722
--- /dev/null
+++ b/src/bin/pg_dump/filter.c
@@ -0,0 +1,472 @@
+/*-------------------------------------------------------------------------
+ *
+ * Implementation of simple filter file parser
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/fe_memutils.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "filter.h"
+#include "lib/stringinfo.h"
+#include "pqexpbuffer.h"
+
+#define is_keyword_str(cstr, str, bytes) \
+ ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0))
+
+/*
+ * Following routines are called from pg_dump, pg_dumpall and pg_restore.
+ * Unfortunately, the implementation of exit_nicely in pg_dump and pg_restore is
+ * different from the one in pg_dumpall, so instead of calling exit_nicely we
+ * have to return some error flag (in this case NULL), and exit_nicely will be
+ * executed from caller's routine.
+ */
+
+/*
+ * Opens filter's file and initialize fstate structure.
+ * Returns true on success.
+ */
+void
+filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit)
+{
+ fstate->filename = filename;
+ fstate->lineno = 0;
+ fstate->exit_nicely = f_exit;
+ initStringInfo(&fstate->linebuff);
+
+ if (strcmp(filename, "-") != 0)
+ {
+ fstate->fp = fopen(filename, "r");
+ if (!fstate->fp)
+ {
+ pg_log_error("could not open filter file \"%s\": %m", filename);
+ fstate->exit_nicely(1);
+ }
+ }
+ else
+ fstate->fp = stdin;
+}
+
+/*
+ * Release allocated resources for the given filter.
+ */
+void
+filter_free(FilterStateData *fstate)
+{
+ if (!fstate)
+ return;
+
+ free(fstate->linebuff.data);
+ fstate->linebuff.data = NULL;
+
+ if (fstate->fp && fstate->fp != stdin)
+ {
+ if (fclose(fstate->fp) != 0)
+ pg_log_error("could not close filter file \"%s\": %m", fstate->filename);
+
+ fstate->fp = NULL;
+ }
+}
+
+/*
+ * Translate FilterObjectType enum to string. It is designed for formatting
+ * of error message in log_unsupported_filter_object_type routine.
+ */
+const char *
+filter_object_type_name(FilterObjectType fot)
+{
+ switch (fot)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ return "comment or empty line";
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ return "table data";
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ return "table data and children";
+ case FILTER_OBJECT_TYPE_DATABASE:
+ return "database";
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ return "extension";
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ return "foreign data";
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ return "function";
+ case FILTER_OBJECT_TYPE_INDEX:
+ return "index";
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ return "schema";
+ case FILTER_OBJECT_TYPE_TABLE:
+ return "table";
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ return "table and children";
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ return "trigger";
+ }
+
+ /* should never get here */
+ pg_unreachable();
+}
+
+/*
+ * Returns true when keyword is one of supported object types, and
+ * set related objtype. Returns false, when keyword is not assigned
+ * with known object type.
+ */
+static bool
+get_object_type(const char *keyword, int size, FilterObjectType *objtype)
+{
+ if (is_keyword_str("table_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA;
+ else if (is_keyword_str("table_data_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN;
+ else if (is_keyword_str("database", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_DATABASE;
+ else if (is_keyword_str("extension", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_EXTENSION;
+ else if (is_keyword_str("foreign_data", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA;
+ else if (is_keyword_str("function", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_FUNCTION;
+ else if (is_keyword_str("index", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_INDEX;
+ else if (is_keyword_str("schema", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_SCHEMA;
+ else if (is_keyword_str("table", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE;
+ else if (is_keyword_str("table_and_children", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN;
+ else if (is_keyword_str("trigger", keyword, size))
+ *objtype = FILTER_OBJECT_TYPE_TRIGGER;
+ else
+ return false;
+
+ return true;
+}
+
+
+void
+pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+{
+ va_list argp;
+ char buf[256];
+
+ va_start(argp, fmt);
+ vsnprintf(buf, sizeof(buf), fmt, argp);
+ va_end(argp);
+
+ pg_log_error("invalid format in filter \"%s\" on line %d: %s",
+ (fstate->fp == stdin ? "stdin" : fstate->filename),
+ fstate->lineno,
+ buf);
+}
+
+/*
+ * filter_get_keyword - read the next filter keyword from buffer
+ *
+ * Search for keywords (limited to ascii alphabetic characters) in
+ * the passed in line buffer. Returns NULL when the buffer is empty or the first
+ * char is not alpha. The char '_' is allowed, except as the first character.
+ * The length of the found keyword is returned in the size parameter.
+ */
+static const char *
+filter_get_keyword(const char **line, int *size)
+{
+ const char *ptr = *line;
+ const char *result = NULL;
+
+ /* Set returnlength preemptively in case no keyword is found */
+ *size = 0;
+
+ /* Skip initial whitespace */
+ while (isspace(*ptr))
+ ptr++;
+
+ if (isalpha(*ptr))
+ {
+ result = ptr++;
+
+ while (isalpha(*ptr) || *ptr == '_')
+ ptr++;
+
+ *size = ptr - result;
+ }
+
+ *line = ptr;
+
+ return result;
+}
+
+/*
+ * read_quoted_pattern - read quoted possibly multi line string
+ *
+ * Reads a quoted string which can span over multiple lines and returns a
+ * pointer to next char after ending double quotes; it will exit on errors.
+ */
+static const char *
+read_quoted_string(FilterStateData *fstate,
+ const char *str,
+ PQExpBuffer pattern)
+{
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ while (1)
+ {
+ /*
+ * We can ignore \r or \n chars because the string is read by
+ * pg_get_line_buf, so these chars should be just trailing chars.
+ */
+ if (*str == '\r' || *str == '\n')
+ {
+ str++;
+ continue;
+ }
+
+ if (*str == '\0')
+ {
+ Assert(fstate->linebuff.data);
+
+ if (!pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ if (ferror(fstate->fp))
+ pg_log_error("could not read from filter file \"%s\": %m",
+ fstate->filename);
+ else
+ pg_log_filter_error(fstate, _("unexpected end of file"));
+
+ fstate->exit_nicely(1);
+ }
+
+ str = fstate->linebuff.data;
+
+ appendPQExpBufferChar(pattern, '\n');
+ fstate->lineno++;
+ }
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+
+ if (*str == '"')
+ {
+ appendPQExpBufferChar(pattern, '"');
+ str++;
+ }
+ else
+ break;
+ }
+ else if (*str == '\\')
+ {
+ str++;
+ if (*str == 'n')
+ appendPQExpBufferChar(pattern, '\n');
+ else if (*str == '\\')
+ appendPQExpBufferChar(pattern, '\\');
+
+ str++;
+ }
+ else
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ return str;
+}
+
+/*
+ * read_pattern - reads on object pattern from input
+ *
+ * This function will parse any valid identifier (quoted or not, qualified or
+ * not), which can also includes the full signature for routines.
+ * Note that this function takes special care to sanitize the detected
+ * identifier (removing extraneous whitespaces or other unnecessary
+ * characters). This is necessary as most backup/restore filtering functions
+ * only recognize identifiers if they are written exactly the same way as
+ * they are output by the server.
+ *
+ * Returns a pointer to next character after the found identifier and exits
+ * on error.
+ */
+static const char *
+read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern)
+{
+ bool skip_space = true;
+ bool found_space = false;
+
+ /* Skip initial whitespace */
+ while (isspace(*str))
+ str++;
+
+ if (*str == '\0')
+ {
+ pg_log_filter_error(fstate, _("missing object name pattern"));
+ fstate->exit_nicely(1);
+ }
+
+ while (*str && *str != '#')
+ {
+ while (*str && !isspace(*str) && !strchr("#,.()\"", *str))
+ {
+ /*
+ * Append space only when it is allowed, and when it was found in
+ * original string.
+ */
+ if (!skip_space && found_space)
+ {
+ appendPQExpBufferChar(pattern, ' ');
+ skip_space = true;
+ }
+
+ appendPQExpBufferChar(pattern, *str++);
+ }
+
+ skip_space = false;
+
+ if (*str == '"')
+ {
+ if (found_space)
+ appendPQExpBufferChar(pattern, ' ');
+
+ str = read_quoted_string(fstate, str, pattern);
+ }
+ else if (*str == ',')
+ {
+ appendPQExpBufferStr(pattern, ", ");
+ skip_space = true;
+ str++;
+ }
+ else if (*str && strchr(".()", *str))
+ {
+ appendPQExpBufferChar(pattern, *str++);
+ skip_space = true;
+ }
+
+ found_space = false;
+
+ /* skip ending whitespaces */
+ while (isspace(*str))
+ {
+ found_space = true;
+ str++;
+ }
+ }
+
+ return str;
+}
+
+/*
+ * filter_read_item - Read command/type/pattern triplet from a filter file
+ *
+ * This will parse one filter item from the filter file, and while it is a
+ * row based format a pattern may span more than one line due to how object
+ * names can be constructed. The expected format of the filter file is:
+ *
+ * <command> <object_type> <pattern>
+ *
+ * command can be "include" or "exclude".
+ *
+ * Supported object types are described by enum FilterObjectType
+ * (see function get_object_type).
+ *
+ * pattern can be any possibly-quoted and possibly-qualified identifier. It
+ * follows the same rules as other object include and exclude functions so it
+ * can also use wildcards.
+ *
+ * Returns true when one filter item was successfully read and parsed. When
+ * object name contains \n chars, then more than one line from input file can
+ * be processed. Returns false when the filter file reaches EOF. In case of
+ * error, the function will emit an appropriate error message before returning
+ * false.
+ */
+bool
+filter_read_item(FilterStateData *fstate,
+ char **objname,
+ FilterCommandType *comtype,
+ FilterObjectType *objtype)
+{
+ if (pg_get_line_buf(fstate->fp, &fstate->linebuff))
+ {
+ const char *str = fstate->linebuff.data;
+ const char *keyword;
+ int size;
+ PQExpBufferData pattern;
+
+ fstate->lineno++;
+
+ /* Skip initial white spaces */
+ while (isspace(*str))
+ str++;
+
+ /*
+ * Skip empty lines or lines where the first non-whitespace character
+ * is a hash indicating a comment.
+ */
+ if (*str != '\0' && *str != '#')
+ {
+ /*
+ * First we expect sequence of two keywords, {include|exclude}
+ * followed by the object type to operate on.
+ */
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate,
+ _("no filter command found (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ if (is_keyword_str("include", keyword, size))
+ *comtype = FILTER_COMMAND_TYPE_INCLUDE;
+ else if (is_keyword_str("exclude", keyword, size))
+ *comtype = FILTER_COMMAND_TYPE_EXCLUDE;
+ else
+ {
+ pg_log_filter_error(fstate,
+ _("invalid filter command (expected \"include\" or \"exclude\")"));
+ fstate->exit_nicely(1);
+ }
+
+ keyword = filter_get_keyword(&str, &size);
+ if (!keyword)
+ {
+ pg_log_filter_error(fstate, _("missing filter object type"));
+ fstate->exit_nicely(1);
+ }
+
+ if (!get_object_type(keyword, size, objtype))
+ {
+ pg_log_filter_error(fstate,
+ _("unsupported filter object type: \"%.*s\""), size, keyword);
+ fstate->exit_nicely(1);
+ }
+
+ initPQExpBuffer(&pattern);
+
+ str = read_pattern(fstate, str, &pattern);
+ *objname = pattern.data;
+ }
+ else
+ {
+ *objname = NULL;
+ *comtype = FILTER_COMMAND_TYPE_NONE;
+ *objtype = FILTER_OBJECT_TYPE_NONE;
+ }
+
+ return true;
+ }
+
+ if (ferror(fstate->fp))
+ {
+ pg_log_error("could not read from filter file \"%s\": %m", fstate->filename);
+ fstate->exit_nicely(1);
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h
new file mode 100644
index 0000000000..4c12f8a572
--- /dev/null
+++ b/src/bin/pg_dump/filter.h
@@ -0,0 +1,70 @@
+/*-------------------------------------------------------------------------
+ *
+ * filter.h
+ * Common header file for the parser of filter file
+ *
+ * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/bin/pg_dump/filter.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "lib/stringinfo.h"
+
+/* Function signature for exit_nicely functions */
+typedef void (*exit_function) (int status);
+
+/*
+ * State data for reading filter items from stream
+ */
+typedef struct
+{
+ FILE *fp;
+ const char *filename;
+ exit_function exit_nicely;
+ int lineno;
+ StringInfoData linebuff;
+} FilterStateData;
+
+/*
+ * List of command types that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_COMMAND_TYPE_NONE,
+ FILTER_COMMAND_TYPE_INCLUDE,
+ FILTER_COMMAND_TYPE_EXCLUDE,
+} FilterCommandType;
+
+/*
+ * List of objects that can be specified in filter file
+ */
+typedef enum
+{
+ FILTER_OBJECT_TYPE_NONE,
+ FILTER_OBJECT_TYPE_TABLE_DATA,
+ FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_DATABASE,
+ FILTER_OBJECT_TYPE_EXTENSION,
+ FILTER_OBJECT_TYPE_FOREIGN_DATA,
+ FILTER_OBJECT_TYPE_FUNCTION,
+ FILTER_OBJECT_TYPE_INDEX,
+ FILTER_OBJECT_TYPE_SCHEMA,
+ FILTER_OBJECT_TYPE_TABLE,
+ FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN,
+ FILTER_OBJECT_TYPE_TRIGGER,
+} FilterObjectType;
+
+extern const char *filter_object_type_name(FilterObjectType fot);
+extern void filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit);
+extern void filter_free(FilterStateData *fstate);
+extern void pg_log_filter_error(FilterStateData *fstate, const char *fmt,...)
+ pg_attribute_printf(2, 3);
+extern bool filter_read_item(FilterStateData *fstate, char **objname,
+ FilterCommandType *comtype,FilterObjectType *objtype);
+
+#endif
diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build
index 9d59a106f3..b6603e26a5 100644
--- a/src/bin/pg_dump/meson.build
+++ b/src/bin/pg_dump/meson.build
@@ -7,6 +7,7 @@ pg_dump_common_sources = files(
'compress_none.c',
'compress_zstd.c',
'dumputils.c',
+ 'filter.c',
'parallel.c',
'pg_backup_archiver.c',
'pg_backup_custom.c',
@@ -99,6 +100,7 @@ tests += {
't/002_pg_dump.pl',
't/003_pg_dump_with_server.pl',
't/004_pg_dump_parallel.pl',
+ 't/005_pg_dump_filterfile.pl',
't/010_dump_connstr.pl',
],
},
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 34fd0a86e9..1e7756beba 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -60,6 +60,7 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout);
static void setupDumpWorker(Archive *AH);
static TableInfo *getRootTableInfo(const TableInfo *tbinfo);
static bool forcePartitionRootLoad(const TableInfo *tbinfo);
+static void read_dump_filters(const char *filename, DumpOptions *dopt);
int
@@ -433,7 +435,7 @@ main(int argc, char **argv)
{"exclude-table-and-children", required_argument, NULL, 13},
{"exclude-table-data-and-children", required_argument, NULL, 14},
{"sync-method", required_argument, NULL, 15},
-
+ {"filter", required_argument, NULL, 16},
{NULL, 0, NULL, 0}
};
@@ -664,6 +666,10 @@ main(int argc, char **argv)
exit_nicely(1);
break;
+ case 16: /* object filters from file */
+ read_dump_filters(optarg, &dopt);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -1111,6 +1117,8 @@ help(const char *progname)
" do NOT dump data for the specified table(s),\n"
" including child and partition tables\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME dump objects and data based on the filter expressions\n"
+ " in specified file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --include-foreign-data=PATTERN\n"
" include data of foreign tables on foreign\n"
@@ -18771,3 +18779,112 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions,
if (!res)
pg_log_warning("could not parse %s array", "reloptions");
}
+
+/*
+ * read_dump_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_dump_filters(const char *filename, DumpOptions *dopt)
+{
+ FilterStateData fstate;
+ char *objname;
+ FilterCommandType comtype;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &objname, &comtype, &objtype))
+ {
+ if (comtype == FILTER_COMMAND_TYPE_INCLUDE)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break; /* unreachable */
+
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ simple_string_list_append(&extension_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ simple_string_list_append(&foreign_servers_include_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_include_patterns, objname);
+ dopt->include_everything = false;
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_include_patterns_and_children,
+ objname);
+ dopt->include_everything = false;
+ break;
+ }
+ }
+ else if (comtype == FILTER_COMMAND_TYPE_EXCLUDE)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ simple_string_list_append(&tabledata_exclude_patterns,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ simple_string_list_append(&tabledata_exclude_patterns_and_children,
+ objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&schema_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ simple_string_list_append(&table_exclude_patterns, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ simple_string_list_append(&table_exclude_patterns_and_children,
+ objname);
+ break;
+ }
+ }
+ else
+ {
+ Assert(comtype == FILTER_COMMAND_TYPE_NONE);
+ Assert(objtype == FILTER_OBJECT_TYPE_NONE);
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index e2a9733d34..fe2a541c56 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -26,6 +26,7 @@
#include "common/string.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "pg_backup.h"
@@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query);
static void executeCommand(PGconn *conn, const char *query);
static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns,
SimpleStringList *names);
+static void read_dumpall_filters(const char *filename, SimpleStringList *patterns);
static char pg_dump_bin[MAXPGPATH];
static const char *progname;
@@ -158,6 +160,7 @@ main(int argc, char *argv[])
{"disable-triggers", no_argument, &disable_triggers, 1},
{"exclude-database", required_argument, NULL, 6},
{"extra-float-digits", required_argument, NULL, 5},
+ {"filter", required_argument, NULL, 8},
{"if-exists", no_argument, &if_exists, 1},
{"inserts", no_argument, &inserts, 1},
{"lock-wait-timeout", required_argument, NULL, 2},
@@ -360,6 +363,10 @@ main(int argc, char *argv[])
appendShellString(pgdumpopts, optarg);
break;
+ case 8:
+ read_dumpall_filters(optarg, &database_exclude_patterns);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -653,6 +660,7 @@ help(void)
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n"));
printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n"));
+ printf(_(" --filter=FILENAME exclude databases specified in filter file\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
@@ -1913,7 +1921,6 @@ executeCommand(PGconn *conn, const char *query)
PQclear(res);
}
-
/*
* dumpTimestamp
*/
@@ -1937,3 +1944,62 @@ hash_string_pointer(char *s)
return hash_bytes(ss, strlen(s));
}
+
+/*
+ * read_dumpall_filters - retrieve database identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ *
+ * At the moment, the only allowed filter is for database exclusion.
+ */
+static void
+read_dumpall_filters(const char *filename, SimpleStringList *pattern)
+{
+ FilterStateData fstate;
+ char *objname;
+ FilterCommandType comtype;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit);
+
+ while (filter_read_item(&fstate, &objname, &comtype, &objtype))
+ {
+ if (comtype == FILTER_COMMAND_TYPE_INCLUDE)
+ {
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+ }
+
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ pg_log_filter_error(&fstate, _("unsupported filter object."));
+ exit_nicely(1);
+ break;
+
+ case FILTER_OBJECT_TYPE_DATABASE:
+ simple_string_list_append(pattern, objname);
+ break;
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 049a100634..0cca9ee612 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -47,11 +47,13 @@
#include "dumputils.h"
#include "fe_utils/option_utils.h"
+#include "filter.h"
#include "getopt_long.h"
#include "parallel.h"
#include "pg_backup_utils.h"
static void usage(const char *progname);
+static void read_restore_filters(const char *filename, RestoreOptions *dopt);
int
main(int argc, char **argv)
@@ -123,6 +125,7 @@ main(int argc, char **argv)
{"no-publications", no_argument, &no_publications, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -286,6 +289,10 @@ main(int argc, char **argv)
set_dump_section(optarg, &(opts->dumpSections));
break;
+ case 4:
+ read_restore_filters(optarg, opts);
+ break;
+
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -463,6 +470,7 @@ usage(const char *progname)
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
+ printf(_(" --filter=FILE restore objects based on filter expressions\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comments\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
@@ -494,3 +502,103 @@ usage(const char *progname)
printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT);
printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
}
+
+/*
+ * read_restore_filters - retrieve object identifier patterns from file
+ *
+ * Parse the specified filter file for include and exclude patterns, and add
+ * them to the relevant lists. If the filename is "-" then filters will be
+ * read from STDIN rather than a file.
+ */
+static void
+read_restore_filters(const char *filename, RestoreOptions *opts)
+{
+ FilterStateData fstate;
+ char *objname;
+ FilterCommandType comtype;
+ FilterObjectType objtype;
+
+ filter_init(&fstate, filename, exit_nicely);
+
+ while (filter_read_item(&fstate, &objname, &comtype, &objtype))
+ {
+ if (comtype == FILTER_COMMAND_TYPE_INCLUDE)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "include",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ opts->selTypes = 1;
+ opts->selFunction = 1;
+ simple_string_list_append(&opts->functionNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_INDEX:
+ opts->selTypes = 1;
+ opts->selIndex = 1;
+ simple_string_list_append(&opts->indexNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TABLE:
+ opts->selTypes = 1;
+ opts->selTable = 1;
+ simple_string_list_append(&opts->tableNames, objname);
+ break;
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ opts->selTypes = 1;
+ opts->selTrigger = 1;
+ simple_string_list_append(&opts->triggerNames, objname);
+ break;
+ }
+ }
+ else if (comtype == FILTER_COMMAND_TYPE_EXCLUDE)
+ {
+ switch (objtype)
+ {
+ case FILTER_OBJECT_TYPE_NONE:
+ break;
+ case FILTER_OBJECT_TYPE_TABLE_DATA:
+ case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_DATABASE:
+ case FILTER_OBJECT_TYPE_EXTENSION:
+ case FILTER_OBJECT_TYPE_FOREIGN_DATA:
+ case FILTER_OBJECT_TYPE_FUNCTION:
+ case FILTER_OBJECT_TYPE_INDEX:
+ case FILTER_OBJECT_TYPE_TABLE:
+ case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN:
+ case FILTER_OBJECT_TYPE_TRIGGER:
+ pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."),
+ "exclude",
+ filter_object_type_name(objtype));
+ exit_nicely(1);
+
+ case FILTER_OBJECT_TYPE_SCHEMA:
+ simple_string_list_append(&opts->schemaExcludeNames, objname);
+ break;
+ }
+ }
+ else
+ {
+ Assert(comtype == FILTER_COMMAND_TYPE_NONE);
+ Assert(objtype == FILTER_OBJECT_TYPE_NONE);
+ }
+
+ if (objname)
+ free(objname);
+ }
+
+ filter_free(&fstate);
+}
diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
new file mode 100644
index 0000000000..09d3262b8b
--- /dev/null
+++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl
@@ -0,0 +1,773 @@
+
+# Copyright (c) 2021, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More tests => 106;
+
+my $tempdir = PostgreSQL::Test::Utils::tempdir;;
+my $inputfile;
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+my $port = $node->port;
+my $backupdir = $node->backup_dir;
+my $plainfile = "$backupdir/plain.sql";
+
+$node->init;
+$node->start;
+
+# Generate test objects
+$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;');
+$node->safe_psql('postgres',
+ 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;');
+
+$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)");
+$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"strange aaa
+name\"(a varchar)");
+$node->safe_psql(
+ 'postgres', "CREATE TABLE \"
+t
+t
+\"(a int)");
+
+$node->safe_psql('postgres',
+ "INSERT INTO table_one VALUES('*** TABLE ONE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_two VALUES('*** TABLE TWO ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three VALUES('*** TABLE THREE ***')");
+$node->safe_psql('postgres',
+ "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')");
+$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)");
+
+$node->safe_psql('postgres', "CREATE DATABASE sourcedb");
+$node->safe_psql('postgres', "CREATE DATABASE targetdb");
+
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql');
+$node->safe_psql('sourcedb', 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s1');
+$node->safe_psql('sourcedb', 'CREATE SCHEMA s2');
+$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)');
+$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1');
+$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)');
+$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+$node->safe_psql('sourcedb', 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()');
+
+#
+# Test interaction of correctly specified filter file
+#
+my ($cmd, $stdout, $stderr, $result);
+
+# Empty filterfile
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "\n # a comment and nothing more\n\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+my $dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, "table three one dumped");
+
+# Test various combinations of whitespace, comments and correct filters
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile " include table table_one #comment\n";
+print $inputfile "include table table_two\n";
+print $inputfile "# skip this line\n";
+print $inputfile "\n";
+print $inputfile "\t\n";
+print $inputfile " \t# another comment\n";
+print $inputfile "exclude table_data table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter patterns as well as comments and whitespace");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m,
+ "table three_one not dumped");
+ok( $dump !~ qr/^COPY public\.table_one/m,
+ "content of table one is not included");
+ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included");
+
+# Test dumping tables specified by qualified names
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table public.table_one\n";
+print $inputfile "include table \"public\".\"table_two\"\n";
+print $inputfile "include table \"public\". table_three\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with exclusion of a single table");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping tables with a wildcard pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_thre*\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with wildcard in pattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped");
+ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped");
+ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three");
+ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m,
+ "dumped table three_one");
+
+# Test dumping table with multiline quoted tablename
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"strange aaa
+name\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with multiline names requiring quoting");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding multiline quoted tablename from dump
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table \"strange aaa\\nname\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m,
+ "dump table with new line in name");
+
+# Test excluding an entire schema
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema public\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "exclude the public schema");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test including and excluding an entire schema by multiple filterfiles
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema public\n";
+close $inputfile;
+
+open my $alt_inputfile, '>', "$tempdir/inputfile2.txt"
+ or die "unable to open filterfile for writing";
+print $alt_inputfile "exclude schema public\n";
+close $alt_inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "--filter=$tempdir/inputfile2.txt", 'postgres'
+ ],
+ "exclude the public schema with multiple filters");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE/m, "no table dumped");
+
+# Test dumping a table with a single leading newline on a row
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"
+t
+t
+\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table \"\\nt\\nt\\n\"";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms,
+ "dump table with multiline strange name");
+
+#########################################
+# Test foreign_data
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include foreign_data doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching foreign servers were found for pattern/,
+ "dump nonexisting foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile, "include foreign_data dummyserver\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "dump foreign_data with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude foreign_data dummy*\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/exclude filter for "foreign data" is not allowed/,
+ "erroneously exclude foreign server");
+
+#########################################
+# Test broken input format
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data"
+);
+
+# Test missing object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/missing object name/,
+ "invalid syntax: missing object identifier pattern");
+
+# Test adding extra content after the object identifier pattern
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table one";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "invalid syntax: extra content after object identifier pattern");
+
+#########################################
+# Combined with --strict-names
+
+# First ensure that a matching filter works
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_one\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ "strict names with matching mattern");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped");
+
+# Now append a pattern to the filter file which doesn't resolve
+open $inputfile, '>>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_nonexisting_name";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ '--strict-names', 'postgres'
+ ],
+ qr/no matching tables were found/,
+ "inclusion of non-existing objects with --strict names");
+
+#########################################
+# pg_dumpall tests
+
+###########################
+# Test dumping all tables except one
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude database postgres\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ "dump tables with exclusion of a database");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped");
+ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped");
+
+# Test invalid filter command
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "k";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/invalid filter command/,
+ "invalid syntax: incorrect filter command");
+
+# Test invalid object type
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/unsupported filter object type: "xxx"/,
+ "invalid syntax: exclusion of non-existing object type"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table foo";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dumpall', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/pg_dumpall: error: invalid format in filter/,
+ "invalid syntax: exclusion of unsupported object type"
+);
+
+#########################################
+# pg_restore tests
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'postgres'
+ ],
+ "dump all tables");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table table_two";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore tables with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored");
+ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include table_data xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/include filter for "table data" is not allowed/,
+ "invalid syntax: inclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/include filter for "extension" is not allowed/,
+ "invalid syntax: inclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude extension xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/exclude filter for "extension" is not allowed/,
+ "invalid syntax: exclusion of non allowed object"
+);
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude table_data xxx";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt"
+ ],
+ qr/xclude filter for "table data" is not allowed/,
+ "invalid syntax: exclusion of non allowed object"
+);
+
+#########################################
+# test restore of other objects
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump",
+ "-Fc", 'sourcedb'
+ ],
+ "dump all objects from sourcedb");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo1(integer)";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored");
+ok($dump !~ qr/^CREATE TABLE public\.foo2/m, "unwanted function is not restored");
+
+# this should be white space tolerant (against the -P argument)
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include function foo3 ( double precision , integer) ";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include index t1_idx1\n";
+
+# attention! this hit pg_restore bug - correct name of trigger is "trg1"
+# not "t1 trg1". Should be fixed when pg_restore will be fixed
+print $inputfile "include trigger t1 trg1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored");
+ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored");
+ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored");
+ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored");
+ok($dump =~ qr/^CREATE SEQUENCE s1\.s1/m, "wanted sequence from schema restored");
+ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "exclude schema s1\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_restore', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt",
+ "-Fc", "$tempdir/filter_test.dump"
+ ],
+ "restore function with filter");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE s1\.t1/m, "unwanted table from schema is not restored");
+ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, "unwanted sequence from schema is not restored");
+ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored");
+ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored");
+
+#########################################
+# test of supported syntax
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "include table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump !~ qr/^CREATE TABLE public\.bootab/m, "exclude dumped children table");
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+
+print $inputfile "exclude table_data_and_children footab\n";
+close $inputfile;
+
+command_ok(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ "filter file without patterns");
+
+$dump = slurp_file($plainfile);
+
+ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table");
+ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table");
+
+#########################################
+# Test extension
+
+open $inputfile, '>', "$tempdir/inputfile.txt"
+ or die "unable to open filterfile for writing";
+print $inputfile "include extension doesnt_exists\n";
+close $inputfile;
+
+command_fails_like(
+ [
+ 'pg_dump', '-p', $port, '-f', $plainfile,
+ "--filter=$tempdir/inputfile.txt", 'postgres'
+ ],
+ qr/pg_dump: error: no matching extensions were found/,
+ "dump nonexisting extension");
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 84f648c174..bcbcd8116f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -455,6 +455,7 @@ sub mkvcbuild
$pgdumpall->AddIncludeDir('src/backend');
$pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c');
$pgdumpall->AddFile('src/bin/pg_dump/dumputils.c');
+ $pgdumpall->AddFile('src/bin/pg_dump/filter.c');
$pgdumpall->AddLibrary('ws2_32.lib');
my $pgrestore = AddSimpleFrontend('pg_dump', 1);
--
2.42.0
^ permalink raw reply [nested|flat] 19+ messages in thread
end of thread, other threads:[~2023-11-20 05:20 UTC | newest]
Thread overview: 19+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-06-06 22:42 [PATCH] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]>
2023-03-07 02:47 Re: proposal: possibility to read dumped table's name from file Julien Rouhaud <[email protected]>
2023-03-16 12:05 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-18 19:48 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-19 14:01 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-19 20:27 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-20 07:01 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 15:32 ` Re: proposal: possibility to read dumped table's name from file Justin Pryzby <[email protected]>
2023-03-21 15:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-03-21 22:00 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:34 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 04:57 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-09-11 06:33 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-09 14:26 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-12 13:17 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:15 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-13 13:39 ` Re: proposal: possibility to read dumped table's name from file Daniel Gustafsson <[email protected]>
2023-11-13 16:07 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
2023-11-20 05:20 ` Re: proposal: possibility to read dumped table's name from file Pavel Stehule <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox