agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v3] Provide relfilenode statistics 35+ messages / 2 participants [nested] [flat]
* [PATCH v3] Provide relfilenode statistics @ 2023-11-16 02:30 Bertrand Drouvot <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Bertrand Drouvot @ 2023-11-16 02:30 UTC (permalink / raw) We currently don’t have writes counters for relations. The reason is that we don’t have the relation OID when writing buffers out. Tracking writes per relfilenode would allow us to track/consolidate writes per relation. relfilenode stats is also beneficial for the "Split index and table statistics into different types of stats" work in progress: it would allow us to avoid additional branches in some situations. === Remarks === This is a POC patch. There is still work to do: there is more places we should add relfilenode counters, create more APIS to retrieve the relfilenode stats, the patch takes care of rewrite generated by TRUNCATE but there is more to care about like CLUSTER,VACUUM FULL. The new logic to retrieve stats in pg_statio_all_tables has been implemented only for the new blocks_written stat (we'd need to do the same for the existing buffer read / buffer hit if we agree on the approach implemented here). The goal of this patch is to start the discussion and agree on the design before moving forward. --- src/backend/access/rmgrdesc/xactdesc.c | 5 +- src/backend/catalog/storage.c | 8 ++ src/backend/catalog/system_functions.sql | 2 +- src/backend/catalog/system_views.sql | 5 +- src/backend/postmaster/checkpointer.c | 5 + src/backend/storage/buffer/bufmgr.c | 6 +- src/backend/storage/smgr/md.c | 7 ++ src/backend/utils/activity/pgstat.c | 39 ++++-- src/backend/utils/activity/pgstat_database.c | 12 +- src/backend/utils/activity/pgstat_function.c | 13 +- src/backend/utils/activity/pgstat_relation.c | 112 ++++++++++++++++-- src/backend/utils/activity/pgstat_replslot.c | 13 +- src/backend/utils/activity/pgstat_shmem.c | 19 ++- .../utils/activity/pgstat_subscription.c | 12 +- src/backend/utils/activity/pgstat_xact.c | 60 +++++++--- src/backend/utils/adt/pgstatfuncs.c | 34 +++++- src/include/access/tableam.h | 19 +++ src/include/access/xact.h | 1 + src/include/catalog/pg_proc.dat | 14 ++- src/include/pgstat.h | 37 ++++-- src/include/utils/pgstat_internal.h | 34 ++++-- src/test/recovery/t/029_stats_restart.pl | 40 +++---- .../recovery/t/030_stats_cleanup_replica.pl | 6 +- src/test/regress/expected/rules.out | 12 +- src/test/regress/expected/stats.out | 30 ++--- src/test/regress/sql/stats.sql | 30 ++--- src/test/subscription/t/026_stats.pl | 4 +- src/tools/pgindent/typedefs.list | 1 + 28 files changed, 424 insertions(+), 156 deletions(-) 4.4% src/backend/catalog/ 46.2% src/backend/utils/activity/ 6.3% src/backend/utils/adt/ 3.6% src/backend/ 3.2% src/include/access/ 3.2% src/include/catalog/ 6.0% src/include/utils/ 6.6% src/include/ 11.7% src/test/recovery/t/ 5.3% src/test/regress/expected/ 3.0% src/ diff --git a/src/backend/access/rmgrdesc/xactdesc.c b/src/backend/access/rmgrdesc/xactdesc.c index dccca201e0..c02b079645 100644 --- a/src/backend/access/rmgrdesc/xactdesc.c +++ b/src/backend/access/rmgrdesc/xactdesc.c @@ -319,10 +319,11 @@ xact_desc_stats(StringInfo buf, const char *label, appendStringInfo(buf, "; %sdropped stats:", label); for (i = 0; i < ndropped; i++) { - appendStringInfo(buf, " %d/%u/%u", + appendStringInfo(buf, " %d/%u/%u/%u", dropped_stats[i].kind, dropped_stats[i].dboid, - dropped_stats[i].objoid); + dropped_stats[i].objoid, + dropped_stats[i].relfile); } } } diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index f56b3cc0f2..db6107cd90 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -33,6 +33,7 @@ #include "storage/smgr.h" #include "utils/hsearch.h" #include "utils/memutils.h" +#include "utils/pgstat_internal.h" #include "utils/rel.h" /* GUC variables */ @@ -152,6 +153,7 @@ RelationCreateStorage(RelFileLocator rlocator, char relpersistence, if (needs_wal) log_smgrcreate(&srel->smgr_rlocator.locator, MAIN_FORKNUM); + pgstat_create_transactional(PGSTAT_KIND_RELFILENODE, rlocator.dbOid, rlocator.spcOid, rlocator.relNumber); /* * Add the relation to the list of stuff to delete at abort, if we are * asked to do so. @@ -227,6 +229,8 @@ RelationDropStorage(Relation rel) * for now I'll keep the logic simple. */ + pgstat_drop_transactional(PGSTAT_KIND_RELFILENODE, rel->rd_locator.dbOid, rel->rd_locator.spcOid, rel->rd_locator.relNumber); + RelationCloseSmgr(rel); } @@ -253,6 +257,9 @@ RelationPreserveStorage(RelFileLocator rlocator, bool atCommit) PendingRelDelete *pending; PendingRelDelete *prev; PendingRelDelete *next; + PgStat_SubXactStatus *xact_state; + + xact_state = pgStatXactStack; prev = NULL; for (pending = pendingDeletes; pending != NULL; pending = next) @@ -267,6 +274,7 @@ RelationPreserveStorage(RelFileLocator rlocator, bool atCommit) else pendingDeletes = next; pfree(pending); + PgStat_RemoveRelFileNodeFromDroppedStats(xact_state, rlocator); /* prev does not change */ } else diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql index 623b9539b1..ec60ef72e3 100644 --- a/src/backend/catalog/system_functions.sql +++ b/src/backend/catalog/system_functions.sql @@ -684,7 +684,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_reset_single_function_counters(oid) FROM publ REVOKE EXECUTE ON FUNCTION pg_stat_reset_replication_slot(text) FROM public; -REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, oid) FROM public; +REVOKE EXECUTE ON FUNCTION pg_stat_have_stats(text, oid, oid, oid) FROM public; REVOKE EXECUTE ON FUNCTION pg_stat_reset_subscription_stats(oid) FROM public; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 19cabc9a47..a7b13d877b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -746,6 +746,7 @@ CREATE VIEW pg_statio_all_tables AS C.relname AS relname, pg_stat_get_blocks_fetched(C.oid) - pg_stat_get_blocks_hit(C.oid) AS heap_blks_read, + pg_stat_get_blocks_written(C.oid) + pg_stat_get_relfilenode_blocks_written(d.oid, CASE WHEN C.reltablespace <> 0 THEN C.reltablespace ELSE d.dattablespace END, C.relfilenode) AS heap_blks_written, pg_stat_get_blocks_hit(C.oid) AS heap_blks_hit, I.idx_blks_read AS idx_blks_read, I.idx_blks_hit AS idx_blks_hit, @@ -754,7 +755,7 @@ CREATE VIEW pg_statio_all_tables AS pg_stat_get_blocks_hit(T.oid) AS toast_blks_hit, X.idx_blks_read AS tidx_blks_read, X.idx_blks_hit AS tidx_blks_hit - FROM pg_class C LEFT JOIN + FROM pg_database d, pg_class C LEFT JOIN pg_class T ON C.reltoastrelid = T.oid LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) LEFT JOIN LATERAL ( @@ -771,7 +772,7 @@ CREATE VIEW pg_statio_all_tables AS sum(pg_stat_get_blocks_hit(indexrelid))::bigint AS idx_blks_hit FROM pg_index WHERE indrelid = T.oid ) X ON true - WHERE C.relkind IN ('r', 't', 'm'); + WHERE C.relkind IN ('r', 't', 'm') AND d.datname = current_database(); CREATE VIEW pg_statio_sys_tables AS SELECT * FROM pg_statio_all_tables diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index 199f008bcd..7f13a840c4 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -519,6 +519,11 @@ CheckpointerMain(char *startup_data, size_t startup_data_len) /* Report pending statistics to the cumulative stats system */ pgstat_report_checkpointer(); pgstat_report_wal(true); + /* + * No need to check for transaction state in checkpointer before + * calling pgstat_report_stat(). + */ + pgstat_report_stat(true); /* * If any checkpoint flags have been set, redo the loop to handle the diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 4415ba648a..1ac73f672b 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -1186,9 +1186,9 @@ PinBufferForBlock(Relation rel, * WaitReadBuffers() (so, not for hits, and not for buffers that are * zeroed instead), the per-relation stats always count them. */ - pgstat_count_buffer_read(rel); + pgstat_report_relfilenode_buffer_read(rel); if (*foundPtr) - pgstat_count_buffer_hit(rel); + pgstat_report_relfilenode_buffer_hit(rel); } if (*foundPtr) { @@ -3907,6 +3907,8 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, pgBufferUsage.shared_blks_written++; + pgstat_report_relfilenode_blks_written(reln->smgr_rlocator.locator); + /* * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and * end the BM_IO_IN_PROGRESS state. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 6796756358..5bc5fc65cd 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -1447,12 +1447,16 @@ DropRelationFiles(RelFileLocator *delrels, int ndelrels, bool isRedo) { SMgrRelation *srels; int i; + int not_freed_count = 0; srels = palloc(sizeof(SMgrRelation) * ndelrels); for (i = 0; i < ndelrels; i++) { SMgrRelation srel = smgropen(delrels[i], INVALID_PROC_NUMBER); + if (!pgstat_drop_entry(PGSTAT_KIND_RELFILENODE, delrels[i].dbOid, delrels[i].spcOid, delrels[i].relNumber)) + not_freed_count++; + if (isRedo) { ForkNumber fork; @@ -1463,6 +1467,9 @@ DropRelationFiles(RelFileLocator *delrels, int ndelrels, bool isRedo) srels[i] = srel; } + if (not_freed_count > 0) + pgstat_request_entry_refs_gc(); + smgrdounlinkall(srels, ndelrels, isRedo); for (i = 0; i < ndelrels; i++) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index b2ca3f39b7..035fdb2aa1 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -308,6 +308,19 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .delete_pending_cb = pgstat_relation_delete_pending_cb, }, + [PGSTAT_KIND_RELFILENODE] = { + .name = "relfilenode", + + .fixed_amount = false, + + .shared_size = sizeof(PgStatShared_RelFileNode), + .shared_data_off = offsetof(PgStatShared_RelFileNode, stats), + .shared_data_len = sizeof(((PgStatShared_RelFileNode *) 0)->stats), + .pending_size = sizeof(PgStat_StatRelFileNodeEntry), + + .flush_pending_cb = pgstat_relfilenode_flush_cb, + }, + [PGSTAT_KIND_FUNCTION] = { .name = "function", @@ -717,7 +730,7 @@ pgstat_report_stat(bool force) partial_flush = false; - /* flush database / relation / function / ... stats */ + /* flush database / relation / function / relfilenode / ... stats */ partial_flush |= pgstat_flush_pending_entries(nowait); /* flush IO stats */ @@ -797,7 +810,7 @@ pgstat_reset_counters(void) * GRANT system. */ void -pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind); TimestampTz ts = GetCurrentTimestamp(); @@ -806,7 +819,7 @@ pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid) Assert(!pgstat_get_kind_info(kind)->fixed_amount); /* reset the "single counter" */ - pgstat_reset_entry(kind, dboid, objoid, ts); + pgstat_reset_entry(kind, dboid, objoid, relfile, ts); if (!kind_info->accessed_across_databases) pgstat_reset_database_timestamp(dboid, ts); @@ -877,7 +890,7 @@ pgstat_clear_snapshot(void) } void * -pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { PgStat_HashKey key; PgStat_EntryRef *entry_ref; @@ -893,6 +906,7 @@ pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid) key.kind = kind; key.dboid = dboid; key.objoid = objoid; + key.relfile = relfile; /* if we need to build a full snapshot, do so */ if (pgstat_fetch_consistency == PGSTAT_FETCH_CONSISTENCY_SNAPSHOT) @@ -918,7 +932,7 @@ pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid) pgStatLocal.snapshot.mode = pgstat_fetch_consistency; - entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, false, NULL); + entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, relfile, false, NULL); if (entry_ref == NULL || entry_ref->shared_entry->dropped) { @@ -987,13 +1001,13 @@ pgstat_get_stat_snapshot_timestamp(bool *have_snapshot) } bool -pgstat_have_entry(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_have_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { /* fixed-numbered stats always exist */ if (pgstat_get_kind_info(kind)->fixed_amount) return true; - return pgstat_get_entry_ref(kind, dboid, objoid, false, NULL) != NULL; + return pgstat_get_entry_ref(kind, dboid, objoid, relfile, false, NULL) != NULL; } /* @@ -1208,7 +1222,8 @@ pgstat_build_snapshot_fixed(PgStat_Kind kind) * created, false otherwise. */ PgStat_EntryRef * -pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, bool *created_entry) +pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, + RelFileNumber relfile, bool *created_entry) { PgStat_EntryRef *entry_ref; @@ -1223,7 +1238,7 @@ pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, bool *created ALLOCSET_SMALL_SIZES); } - entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, + entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, relfile, true, created_entry); if (entry_ref->pending == NULL) @@ -1246,11 +1261,11 @@ pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, bool *created * that it shouldn't be needed. */ PgStat_EntryRef * -pgstat_fetch_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_fetch_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { PgStat_EntryRef *entry_ref; - entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, false, NULL); + entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, relfile, false, NULL); if (entry_ref == NULL || entry_ref->pending == NULL) return NULL; @@ -1279,7 +1294,7 @@ pgstat_delete_pending_entry(PgStat_EntryRef *entry_ref) } /* - * Flush out pending stats for database objects (databases, relations, + * Flush out pending stats for database objects (databases, relations, relfilenodes, * functions). */ static bool diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 29bc090974..cf77f2dbdb 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -43,7 +43,7 @@ static PgStat_Counter pgLastSessionReportTime = 0; void pgstat_drop_database(Oid databaseid) { - pgstat_drop_transactional(PGSTAT_KIND_DATABASE, databaseid, InvalidOid); + pgstat_drop_transactional(PGSTAT_KIND_DATABASE, databaseid, InvalidOid, InvalidOid); } /* @@ -66,7 +66,7 @@ pgstat_report_autovac(Oid dboid) * operation so it doesn't matter if we get blocked here a little. */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, - dboid, InvalidOid, false); + dboid, InvalidOid, InvalidOid, false); dbentry = (PgStatShared_Database *) entry_ref->shared_stats; dbentry->stats.last_autovac_time = GetCurrentTimestamp(); @@ -150,7 +150,7 @@ pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount) * common enough for that to be a problem. */ entry_ref = - pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, dboid, InvalidOid, false); + pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, dboid, InvalidOid, InvalidOid, false); sharedent = (PgStatShared_Database *) entry_ref->shared_stats; sharedent->stats.checksum_failures += failurecount; @@ -242,7 +242,7 @@ PgStat_StatDBEntry * pgstat_fetch_stat_dbentry(Oid dboid) { return (PgStat_StatDBEntry *) - pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid); + pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid, InvalidOid); } void @@ -341,7 +341,7 @@ pgstat_prep_database_pending(Oid dboid) Assert(!OidIsValid(dboid) || OidIsValid(MyDatabaseId)); entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid, - NULL); + InvalidOid, NULL); return entry_ref->pending; } @@ -357,7 +357,7 @@ pgstat_reset_database_timestamp(Oid dboid, TimestampTz ts) PgStatShared_Database *dbentry; dbref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, MyDatabaseId, InvalidOid, - false); + InvalidOid, false); dbentry = (PgStatShared_Database *) dbref->shared_stats; dbentry->stats.stat_reset_timestamp = ts; diff --git a/src/backend/utils/activity/pgstat_function.c b/src/backend/utils/activity/pgstat_function.c index d26da551a4..440e44e300 100644 --- a/src/backend/utils/activity/pgstat_function.c +++ b/src/backend/utils/activity/pgstat_function.c @@ -46,7 +46,8 @@ pgstat_create_function(Oid proid) { pgstat_create_transactional(PGSTAT_KIND_FUNCTION, MyDatabaseId, - proid); + proid, + InvalidOid); } /* @@ -61,7 +62,8 @@ pgstat_drop_function(Oid proid) { pgstat_drop_transactional(PGSTAT_KIND_FUNCTION, MyDatabaseId, - proid); + proid, + InvalidOid); } /* @@ -86,6 +88,7 @@ pgstat_init_function_usage(FunctionCallInfo fcinfo, entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, fcinfo->flinfo->fn_oid, + InvalidOid, &created_entry); /* @@ -113,7 +116,7 @@ pgstat_init_function_usage(FunctionCallInfo fcinfo, if (!SearchSysCacheExists1(PROCOID, ObjectIdGetDatum(fcinfo->flinfo->fn_oid))) { pgstat_drop_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, - fcinfo->flinfo->fn_oid); + fcinfo->flinfo->fn_oid, InvalidOid); ereport(ERROR, errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("function call to dropped function")); } @@ -224,7 +227,7 @@ find_funcstat_entry(Oid func_id) { PgStat_EntryRef *entry_ref; - entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id); + entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id, InvalidOid); if (entry_ref) return entry_ref->pending; @@ -239,5 +242,5 @@ PgStat_StatFuncEntry * pgstat_fetch_stat_funcentry(Oid func_id) { return (PgStat_StatFuncEntry *) - pgstat_fetch_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id); + pgstat_fetch_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id, InvalidOid); } diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 8a3f7d434c..136dd6c85b 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -44,6 +44,7 @@ typedef struct TwoPhasePgStatRecord static PgStat_TableStatus *pgstat_prep_relation_pending(Oid rel_id, bool isshared); +PgStat_StatRelFileNodeEntry *pgstat_prep_relfilenode_pending(RelFileLocator locator); static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int nest_level); static void ensure_tabstat_xact_level(PgStat_TableStatus *pgstat_info); static void save_truncdrop_counters(PgStat_TableXactStatus *trans, bool is_drop); @@ -69,6 +70,7 @@ pgstat_copy_relation_stats(Relation dst, Relation src) dst_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dst->rd_rel->relisshared ? InvalidOid : MyDatabaseId, RelationGetRelid(dst), + InvalidOid, false); dstshstats = (PgStatShared_Relation *) dst_ref->shared_stats; @@ -170,7 +172,7 @@ pgstat_create_relation(Relation rel) { pgstat_create_transactional(PGSTAT_KIND_RELATION, rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId, - RelationGetRelid(rel)); + RelationGetRelid(rel), InvalidOid); } /* @@ -184,7 +186,7 @@ pgstat_drop_relation(Relation rel) pgstat_drop_transactional(PGSTAT_KIND_RELATION, rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId, - RelationGetRelid(rel)); + RelationGetRelid(rel), InvalidOid); if (!pgstat_should_count_relation(rel)) return; @@ -225,7 +227,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, - dboid, tableoid, false); + dboid, tableoid, InvalidOid, false); shtabentry = (PgStatShared_Relation *) entry_ref->shared_stats; tabentry = &shtabentry->stats; @@ -318,6 +320,7 @@ pgstat_report_analyze(Relation rel, /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, RelationGetRelid(rel), + InvalidOid, false); /* can't get dropped while accessed */ Assert(entry_ref != NULL && entry_ref->shared_stats != NULL); @@ -458,6 +461,19 @@ pgstat_fetch_stat_tabentry(Oid relid) return pgstat_fetch_stat_tabentry_ext(IsSharedRelation(relid), relid); } +/* + * Support function for the SQL-callable pgstat* functions. Returns + * the collected statistics for one relfilenode or NULL. NULL doesn't mean + * that the relfilenode doesn't exist, just that there are no statistics, so the + * caller is better off to report ZERO instead. + */ +PgStat_StatRelFileNodeEntry * +pgstat_fetch_stat_relfilenodeentry(Oid dboid, Oid spcOid, RelFileNumber relfile) +{ + return (PgStat_StatRelFileNodeEntry *) + pgstat_fetch_entry(PGSTAT_KIND_RELFILENODE, dboid, spcOid, relfile); +} + /* * More efficient version of pgstat_fetch_stat_tabentry(), allowing to specify * whether the to-be-accessed table is a shared relation or not. @@ -468,7 +484,7 @@ pgstat_fetch_stat_tabentry_ext(bool shared, Oid reloid) Oid dboid = (shared ? InvalidOid : MyDatabaseId); return (PgStat_StatTabEntry *) - pgstat_fetch_entry(PGSTAT_KIND_RELATION, dboid, reloid); + pgstat_fetch_entry(PGSTAT_KIND_RELATION, dboid, reloid, InvalidOid); } /* @@ -491,10 +507,10 @@ find_tabstat_entry(Oid rel_id) PgStat_TableStatus *tabentry = NULL; PgStat_TableStatus *tablestatus = NULL; - entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_RELATION, MyDatabaseId, rel_id); + entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_RELATION, MyDatabaseId, rel_id, InvalidOid); if (!entry_ref) { - entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_RELATION, InvalidOid, rel_id); + entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_RELATION, InvalidOid, rel_id, InvalidOid); if (!entry_ref) return tablestatus; } @@ -881,6 +897,38 @@ pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) return true; } +/* + * Flush out pending stats for the relfilenode entry + * + * If nowait is true, this function returns false if lock could not + * immediately acquired, otherwise true is returned. + */ +bool +pgstat_relfilenode_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) +{ + PgStatShared_RelFileNode *sharedent; + PgStat_StatRelFileNodeEntry *pendingent; + + pendingent = (PgStat_StatRelFileNodeEntry *) entry_ref->pending; + sharedent = (PgStatShared_RelFileNode *) entry_ref->shared_stats; + + if (!pgstat_lock_entry(entry_ref, nowait)) + return false; + +#define PGSTAT_ACCUM_RELFILENODECOUNT(item) \ + (sharedent)->stats.item += (pendingent)->item + + PGSTAT_ACCUM_RELFILENODECOUNT(blocks_fetched); + PGSTAT_ACCUM_RELFILENODECOUNT(blocks_hit); + PGSTAT_ACCUM_RELFILENODECOUNT(blocks_written); + + pgstat_unlock_entry(entry_ref); + + memset(pendingent, 0, sizeof(*pendingent)); + + return true; +} + void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref) { @@ -902,7 +950,7 @@ pgstat_prep_relation_pending(Oid rel_id, bool isshared) entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_RELATION, isshared ? InvalidOid : MyDatabaseId, - rel_id, NULL); + rel_id, InvalidOid, NULL); pending = entry_ref->pending; pending->id = rel_id; pending->shared = isshared; @@ -910,6 +958,56 @@ pgstat_prep_relation_pending(Oid rel_id, bool isshared) return pending; } +PgStat_StatRelFileNodeEntry * +pgstat_prep_relfilenode_pending(RelFileLocator locator) +{ + PgStat_EntryRef *entry_ref; + + entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_RELFILENODE, locator.dbOid, + locator.spcOid, locator.relNumber, NULL); + + return entry_ref->pending; +} + +void +pgstat_report_relfilenode_blks_written(RelFileLocator locator) +{ + PgStat_StatRelFileNodeEntry *relfileentry = NULL; + + relfileentry = pgstat_prep_relfilenode_pending(locator); + + if (relfileentry) + relfileentry->blocks_written++; +} + +void +pgstat_report_relfilenode_buffer_read(Relation reln) +{ + PgStat_StatRelFileNodeEntry *relfileentry = NULL; + + /* For relation stats to survive after a rewrite */ + pgstat_count_buffer_read(reln); + + relfileentry = pgstat_prep_relfilenode_pending(reln->rd_locator); + + if (relfileentry) + relfileentry->blocks_fetched++; +} + +void +pgstat_report_relfilenode_buffer_hit(Relation reln) +{ + PgStat_StatRelFileNodeEntry *relfileentry = NULL; + + /* For relation stats to survive after a rewrite */ + pgstat_count_buffer_hit(reln); + + relfileentry = pgstat_prep_relfilenode_pending(reln->rd_locator); + + if (relfileentry) + relfileentry->blocks_hit++; +} + /* * add a new (sub)transaction state record */ diff --git a/src/backend/utils/activity/pgstat_replslot.c b/src/backend/utils/activity/pgstat_replslot.c index da11b86744..2e68ed4a09 100644 --- a/src/backend/utils/activity/pgstat_replslot.c +++ b/src/backend/utils/activity/pgstat_replslot.c @@ -62,7 +62,7 @@ pgstat_reset_replslot(const char *name) */ if (SlotIsLogical(slot)) pgstat_reset(PGSTAT_KIND_REPLSLOT, InvalidOid, - ReplicationSlotIndex(slot)); + ReplicationSlotIndex(slot), InvalidOid); LWLockRelease(ReplicationSlotControlLock); } @@ -82,7 +82,7 @@ pgstat_report_replslot(ReplicationSlot *slot, const PgStat_StatReplSlotEntry *re PgStat_StatReplSlotEntry *statent; entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_REPLSLOT, InvalidOid, - ReplicationSlotIndex(slot), false); + ReplicationSlotIndex(slot), InvalidOid, false); shstatent = (PgStatShared_ReplSlot *) entry_ref->shared_stats; statent = &shstatent->stats; @@ -116,7 +116,7 @@ pgstat_create_replslot(ReplicationSlot *slot) Assert(LWLockHeldByMeInMode(ReplicationSlotAllocationLock, LW_EXCLUSIVE)); entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_REPLSLOT, InvalidOid, - ReplicationSlotIndex(slot), false); + ReplicationSlotIndex(slot), InvalidOid, false); shstatent = (PgStatShared_ReplSlot *) entry_ref->shared_stats; /* @@ -146,7 +146,7 @@ void pgstat_acquire_replslot(ReplicationSlot *slot) { pgstat_get_entry_ref(PGSTAT_KIND_REPLSLOT, InvalidOid, - ReplicationSlotIndex(slot), true, NULL); + ReplicationSlotIndex(slot), InvalidOid, true, NULL); } /* @@ -158,7 +158,7 @@ pgstat_drop_replslot(ReplicationSlot *slot) Assert(LWLockHeldByMeInMode(ReplicationSlotAllocationLock, LW_EXCLUSIVE)); if (!pgstat_drop_entry(PGSTAT_KIND_REPLSLOT, InvalidOid, - ReplicationSlotIndex(slot))) + ReplicationSlotIndex(slot), InvalidOid)) pgstat_request_entry_refs_gc(); } @@ -178,7 +178,7 @@ pgstat_fetch_replslot(NameData slotname) if (idx != -1) slotentry = (PgStat_StatReplSlotEntry *) pgstat_fetch_entry(PGSTAT_KIND_REPLSLOT, - InvalidOid, idx); + InvalidOid, idx, InvalidOid); LWLockRelease(ReplicationSlotControlLock); @@ -210,6 +210,7 @@ pgstat_replslot_from_serialized_name_cb(const NameData *name, PgStat_HashKey *ke key->kind = PGSTAT_KIND_REPLSLOT; key->dboid = InvalidOid; key->objoid = idx; + key->relfile = InvalidOid; return true; } diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index fd09b9d988..4890f8b807 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -429,10 +429,10 @@ pgstat_get_entry_ref_cached(PgStat_HashKey key, PgStat_EntryRef **entry_ref_p) * if the entry is newly created, false otherwise. */ PgStat_EntryRef * -pgstat_get_entry_ref(PgStat_Kind kind, Oid dboid, Oid objoid, bool create, - bool *created_entry) +pgstat_get_entry_ref(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile, + bool create, bool *created_entry) { - PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid}; + PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid,.relfile = relfile}; PgStatShared_HashEntry *shhashent; PgStatShared_Common *shheader = NULL; PgStat_EntryRef *entry_ref; @@ -645,12 +645,12 @@ pgstat_unlock_entry(PgStat_EntryRef *entry_ref) */ PgStat_EntryRef * pgstat_get_entry_ref_locked(PgStat_Kind kind, Oid dboid, Oid objoid, - bool nowait) + RelFileNumber relfile, bool nowait) { PgStat_EntryRef *entry_ref; /* find shared table stats entry corresponding to the local entry */ - entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, true, NULL); + entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, relfile, true, NULL); /* lock the shared entry to protect the content, skip if failed */ if (!pgstat_lock_entry(entry_ref, nowait)) @@ -905,9 +905,9 @@ pgstat_drop_database_and_contents(Oid dboid) * pgstat_gc_entry_refs(). */ bool -pgstat_drop_entry(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_drop_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { - PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid}; + PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid,.relfile = relfile}; PgStatShared_HashEntry *shent; bool freed = true; @@ -980,13 +980,12 @@ shared_stat_reset_contents(PgStat_Kind kind, PgStatShared_Common *header, * Reset one variable-numbered stats entry. */ void -pgstat_reset_entry(PgStat_Kind kind, Oid dboid, Oid objoid, TimestampTz ts) +pgstat_reset_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile, TimestampTz ts) { PgStat_EntryRef *entry_ref; Assert(!pgstat_get_kind_info(kind)->fixed_amount); - - entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, false, NULL); + entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, relfile, false, NULL); if (!entry_ref || entry_ref->shared_entry->dropped) return; diff --git a/src/backend/utils/activity/pgstat_subscription.c b/src/backend/utils/activity/pgstat_subscription.c index d9af8de658..9b9ab2861b 100644 --- a/src/backend/utils/activity/pgstat_subscription.c +++ b/src/backend/utils/activity/pgstat_subscription.c @@ -30,7 +30,7 @@ pgstat_report_subscription_error(Oid subid, bool is_apply_error) PgStat_BackendSubEntry *pending; entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_SUBSCRIPTION, - InvalidOid, subid, NULL); + InvalidOid, subid, InvalidOid, NULL); pending = entry_ref->pending; if (is_apply_error) @@ -47,12 +47,12 @@ pgstat_create_subscription(Oid subid) { /* Ensures that stats are dropped if transaction rolls back */ pgstat_create_transactional(PGSTAT_KIND_SUBSCRIPTION, - InvalidOid, subid); + InvalidOid, subid, InvalidOid); /* Create and initialize the subscription stats entry */ - pgstat_get_entry_ref(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, + pgstat_get_entry_ref(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, InvalidOid, true, NULL); - pgstat_reset_entry(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, 0); + pgstat_reset_entry(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, InvalidOid, 0); } /* @@ -64,7 +64,7 @@ void pgstat_drop_subscription(Oid subid) { pgstat_drop_transactional(PGSTAT_KIND_SUBSCRIPTION, - InvalidOid, subid); + InvalidOid, subid, InvalidOid); } /* @@ -75,7 +75,7 @@ PgStat_StatSubEntry * pgstat_fetch_stat_subscription(Oid subid) { return (PgStat_StatSubEntry *) - pgstat_fetch_entry(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid); + pgstat_fetch_entry(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, InvalidOid); } /* diff --git a/src/backend/utils/activity/pgstat_xact.c b/src/backend/utils/activity/pgstat_xact.c index 1877d22f14..b25df5112b 100644 --- a/src/backend/utils/activity/pgstat_xact.c +++ b/src/backend/utils/activity/pgstat_xact.c @@ -30,7 +30,7 @@ static void AtEOXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, bool static void AtEOSubXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, bool isCommit, int nestDepth); -static PgStat_SubXactStatus *pgStatXactStack = NULL; +PgStat_SubXactStatus *pgStatXactStack = NULL; /* @@ -84,7 +84,7 @@ AtEOXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, bool isCommit) * Transaction that dropped an object committed. Drop the stats * too. */ - if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid)) + if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid, it->relfile)) not_freed_count++; } else if (!isCommit && pending->is_create) @@ -93,7 +93,7 @@ AtEOXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, bool isCommit) * Transaction that created an object aborted. Drop the stats * associated with the object. */ - if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid)) + if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid, it->relfile)) not_freed_count++; } @@ -105,6 +105,33 @@ AtEOXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, bool isCommit) pgstat_request_entry_refs_gc(); } +/* + * Remove a relfilenode stat from the list of stats to be dropped. + */ +void +PgStat_RemoveRelFileNodeFromDroppedStats(PgStat_SubXactStatus *xact_state, RelFileLocator rlocator) +{ + dlist_mutable_iter iter; + + if (dclist_count(&xact_state->pending_drops) == 0) + return; + + dclist_foreach_modify(iter, &xact_state->pending_drops) + { + PgStat_PendingDroppedStatsItem *pending = + dclist_container(PgStat_PendingDroppedStatsItem, node, iter.cur); + xl_xact_stats_item *it = &pending->item; + + if (it->kind == PGSTAT_KIND_RELFILENODE && it->dboid == rlocator.dbOid + && it->objoid == rlocator.spcOid && it->relfile == rlocator.relNumber) + { + dclist_delete_from(&xact_state->pending_drops, &pending->node); + pfree(pending); + return; + } + } +} + /* * Called from access/transam/xact.c at subtransaction commit/abort. */ @@ -158,7 +185,7 @@ AtEOSubXact_PgStat_DroppedStats(PgStat_SubXactStatus *xact_state, * Subtransaction creating a new stats object aborted. Drop the * stats object. */ - if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid)) + if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid, it->relfile)) not_freed_count++; pfree(pending); } @@ -320,7 +347,11 @@ pgstat_execute_transactional_drops(int ndrops, struct xl_xact_stats_item *items, { xl_xact_stats_item *it = &items[i]; - if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid)) + /* leave it to pgstat_drop_transactional() in RelationDropStorage() */ + if (it->kind == PGSTAT_KIND_RELFILENODE) + continue; + + if (!pgstat_drop_entry(it->kind, it->dboid, it->objoid, it->relfile)) not_freed_count++; } @@ -329,7 +360,7 @@ pgstat_execute_transactional_drops(int ndrops, struct xl_xact_stats_item *items, } static void -create_drop_transactional_internal(PgStat_Kind kind, Oid dboid, Oid objoid, bool is_create) +create_drop_transactional_internal(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile, bool is_create) { int nest_level = GetCurrentTransactionNestLevel(); PgStat_SubXactStatus *xact_state; @@ -342,6 +373,7 @@ create_drop_transactional_internal(PgStat_Kind kind, Oid dboid, Oid objoid, bool drop->item.kind = kind; drop->item.dboid = dboid; drop->item.objoid = objoid; + drop->item.relfile = relfile; dclist_push_tail(&xact_state->pending_drops, &drop->node); } @@ -354,18 +386,18 @@ create_drop_transactional_internal(PgStat_Kind kind, Oid dboid, Oid objoid, bool * dropped. */ void -pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { - if (pgstat_get_entry_ref(kind, dboid, objoid, false, NULL)) + if (pgstat_get_entry_ref(kind, dboid, objoid, relfile, false, NULL)) { ereport(WARNING, - errmsg("resetting existing statistics for kind %s, db=%u, oid=%u", - (pgstat_get_kind_info(kind))->name, dboid, objoid)); + errmsg("resetting existing statistics for kind %s, db=%u, oid=%u, relfile=%u", + (pgstat_get_kind_info(kind))->name, dboid, objoid, relfile)); - pgstat_reset(kind, dboid, objoid); + pgstat_reset(kind, dboid, objoid, relfile); } - create_drop_transactional_internal(kind, dboid, objoid, /* create */ true); + create_drop_transactional_internal(kind, dboid, objoid, relfile, /* create */ true); } /* @@ -376,7 +408,7 @@ pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid) * alive. */ void -pgstat_drop_transactional(PgStat_Kind kind, Oid dboid, Oid objoid) +pgstat_drop_transactional(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile) { - create_drop_transactional_internal(kind, dboid, objoid, /* create */ false); + create_drop_transactional_internal(kind, dboid, objoid, relfile, /* create */ false); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 3221137123..c1d62873a3 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -106,6 +106,30 @@ PG_STAT_GET_RELENTRY_INT64(tuples_updated) /* pg_stat_get_vacuum_count */ PG_STAT_GET_RELENTRY_INT64(vacuum_count) +#define PG_STAT_GET_RELFILEENTRY_INT64(stat) \ +Datum \ +CppConcat(pg_stat_get_relfilenode_,stat)(PG_FUNCTION_ARGS) \ +{ \ + Oid dboid = PG_GETARG_OID(0); \ + Oid spcOid = PG_GETARG_OID(1); \ + RelFileNumber relfile = PG_GETARG_OID(2); \ + int64 result; \ + PgStat_StatRelFileNodeEntry *relfileentry; \ + \ + if ((relfileentry = pgstat_fetch_stat_relfilenodeentry(dboid, spcOid, relfile)) == NULL) \ + result = 0; \ + else \ + result = (int64) (relfileentry->stat); \ + \ + PG_RETURN_INT64(result); \ +} + +/* pg_stat_get_relfilenode_blocks_written */ +PG_STAT_GET_RELFILEENTRY_INT64(blocks_written) + +/* pg_stat_get_blocks_written */ +PG_STAT_GET_RELENTRY_INT64(blocks_written) + #define PG_STAT_GET_RELENTRY_TIMESTAMPTZ(stat) \ Datum \ CppConcat(pg_stat_get_,stat)(PG_FUNCTION_ARGS) \ @@ -1752,7 +1776,7 @@ pg_stat_reset_single_table_counters(PG_FUNCTION_ARGS) Oid taboid = PG_GETARG_OID(0); Oid dboid = (IsSharedRelation(taboid) ? InvalidOid : MyDatabaseId); - pgstat_reset(PGSTAT_KIND_RELATION, dboid, taboid); + pgstat_reset(PGSTAT_KIND_RELATION, dboid, taboid, InvalidOid); PG_RETURN_VOID(); } @@ -1762,7 +1786,7 @@ pg_stat_reset_single_function_counters(PG_FUNCTION_ARGS) { Oid funcoid = PG_GETARG_OID(0); - pgstat_reset(PGSTAT_KIND_FUNCTION, MyDatabaseId, funcoid); + pgstat_reset(PGSTAT_KIND_FUNCTION, MyDatabaseId, funcoid, InvalidOid); PG_RETURN_VOID(); } @@ -1820,7 +1844,7 @@ pg_stat_reset_subscription_stats(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid subscription OID %u", subid))); - pgstat_reset(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid); + pgstat_reset(PGSTAT_KIND_SUBSCRIPTION, InvalidOid, subid, InvalidOid); } PG_RETURN_VOID(); @@ -2028,7 +2052,9 @@ pg_stat_have_stats(PG_FUNCTION_ARGS) char *stats_type = text_to_cstring(PG_GETARG_TEXT_P(0)); Oid dboid = PG_GETARG_OID(1); Oid objoid = PG_GETARG_OID(2); + Oid relfile = PG_GETARG_OID(3); + PgStat_Kind kind = pgstat_get_kind_from_str(stats_type); - PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objoid)); + PG_RETURN_BOOL(pgstat_have_entry(kind, dboid, objoid, relfile)); } diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index da661289c1..3614bae63c 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -21,7 +21,9 @@ #include "access/sdir.h" #include "access/xact.h" #include "executor/tuptable.h" +#include "pgstat.h" #include "storage/read_stream.h" +#include "utils/pgstat_internal.h" #include "utils/rel.h" #include "utils/snapshot.h" @@ -1624,6 +1626,23 @@ table_relation_set_new_filelocator(Relation rel, TransactionId *freezeXid, MultiXactId *minmulti) { + PgStat_StatRelFileNodeEntry *relfileentry; + PgStat_StatTabEntry *tabentry = NULL; + PgStat_EntryRef *entry_ref = NULL; + PgStatShared_Relation *shtabentry; + + entry_ref = pgstat_get_entry_ref(PGSTAT_KIND_RELATION, MyDatabaseId, rel->rd_id, InvalidOid, false, NULL); + if (entry_ref) + { + shtabentry = (PgStatShared_Relation *) entry_ref->shared_stats; + tabentry = &shtabentry->stats; + } + + relfileentry = pgstat_fetch_stat_relfilenodeentry(rel->rd_locator.dbOid, rel->rd_locator.spcOid, rel->rd_locator.relNumber); + + if (tabentry && relfileentry) + tabentry->blocks_written += relfileentry->blocks_written; + rel->rd_tableam->relation_set_new_filelocator(rel, newrlocator, persistence, freezeXid, minmulti); diff --git a/src/include/access/xact.h b/src/include/access/xact.h index 6d4439f052..3b9ed65ff6 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -284,6 +284,7 @@ typedef struct xl_xact_stats_item int kind; Oid dboid; Oid objoid; + RelFileNumber relfile; } xl_xact_stats_item; typedef struct xl_xact_stats_items diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d36f6001bb..fae89b1e08 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5407,6 +5407,14 @@ proname => 'pg_stat_get_tuples_updated', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_tuples_updated' }, +{ oid => '9280', descr => 'statistics: number of blocks written', + proname => 'pg_stat_get_relfilenode_blocks_written', provolatile => 's', + proparallel => 'r', + proargtypes => 'oid oid oid', + prorettype => 'int8', + proallargtypes => '{oid,oid,oid,int8}', + proargmodes => '{i,i,i,o}', + prosrc => 'pg_stat_get_relfilenode_blocks_written' }, { oid => '1933', descr => 'statistics: number of tuples deleted', proname => 'pg_stat_get_tuples_deleted', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', @@ -5446,6 +5454,10 @@ proname => 'pg_stat_get_blocks_hit', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => 'oid', prosrc => 'pg_stat_get_blocks_hit' }, +{ oid => '8438', descr => 'statistics: number of blocks written', + proname => 'pg_stat_get_blocks_written', provolatile => 's', proparallel => 'r', + prorettype => 'int8', proargtypes => 'oid', + prosrc => 'pg_stat_get_blocks_written' }, { oid => '2781', descr => 'statistics: last manual vacuum time for a table', proname => 'pg_stat_get_last_vacuum_time', provolatile => 's', proparallel => 'r', prorettype => 'timestamptz', proargtypes => 'oid', @@ -5532,7 +5544,7 @@ { oid => '6230', descr => 'statistics: check if a stats object exists', proname => 'pg_stat_have_stats', provolatile => 'v', proparallel => 'r', - prorettype => 'bool', proargtypes => 'text oid oid', + prorettype => 'bool', proargtypes => 'text oid oid oid', prosrc => 'pg_stat_have_stats' }, { oid => '6231', descr => 'statistics: information about subscription stats', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index f63159c55c..cfed4f07f5 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -15,6 +15,7 @@ #include "datatype/timestamp.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ +#include "storage/relfilelocator.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -45,17 +46,18 @@ /* stats for variable-numbered objects */ #define PGSTAT_KIND_DATABASE 1 /* database-wide statistics */ #define PGSTAT_KIND_RELATION 2 /* per-table statistics */ -#define PGSTAT_KIND_FUNCTION 3 /* per-function statistics */ -#define PGSTAT_KIND_REPLSLOT 4 /* per-slot statistics */ -#define PGSTAT_KIND_SUBSCRIPTION 5 /* per-subscription statistics */ +#define PGSTAT_KIND_RELFILENODE 3 /* per-relfilenode statistics */ +#define PGSTAT_KIND_FUNCTION 4 /* per-function statistics */ +#define PGSTAT_KIND_REPLSLOT 5 /* per-slot statistics */ +#define PGSTAT_KIND_SUBSCRIPTION 6 /* per-subscription statistics */ /* stats for fixed-numbered objects */ -#define PGSTAT_KIND_ARCHIVER 6 -#define PGSTAT_KIND_BGWRITER 7 -#define PGSTAT_KIND_CHECKPOINTER 8 -#define PGSTAT_KIND_IO 9 -#define PGSTAT_KIND_SLRU 10 -#define PGSTAT_KIND_WAL 11 +#define PGSTAT_KIND_ARCHIVER 7 +#define PGSTAT_KIND_BGWRITER 8 +#define PGSTAT_KIND_CHECKPOINTER 9 +#define PGSTAT_KIND_IO 10 +#define PGSTAT_KIND_SLRU 11 +#define PGSTAT_KIND_WAL 12 #define PGSTAT_KIND_BUILTIN_MIN PGSTAT_KIND_DATABASE #define PGSTAT_KIND_BUILTIN_MAX PGSTAT_KIND_WAL @@ -447,6 +449,7 @@ typedef struct PgStat_StatTabEntry PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; + PgStat_Counter blocks_written; TimestampTz last_vacuum_time; /* user initiated vacuum */ PgStat_Counter vacuum_count; @@ -458,6 +461,13 @@ typedef struct PgStat_StatTabEntry PgStat_Counter autoanalyze_count; } PgStat_StatTabEntry; +typedef struct PgStat_StatRelFileNodeEntry +{ + PgStat_Counter blocks_fetched; + PgStat_Counter blocks_hit; + PgStat_Counter blocks_written; +} PgStat_StatRelFileNodeEntry; + typedef struct PgStat_WalStats { PgStat_Counter wal_records; @@ -508,7 +518,7 @@ extern long pgstat_report_stat(bool force); extern void pgstat_force_next_flush(void); extern void pgstat_reset_counters(void); -extern void pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid); +extern void pgstat_reset(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); extern void pgstat_reset_of_kind(PgStat_Kind kind); /* stats accessors */ @@ -517,7 +527,7 @@ extern TimestampTz pgstat_get_stat_snapshot_timestamp(bool *have_snapshot); /* helpers */ extern PgStat_Kind pgstat_get_kind_from_str(char *kind_str); -extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, Oid objoid); +extern bool pgstat_have_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); /* @@ -626,6 +636,10 @@ extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); +extern void pgstat_report_relfilenode_blks_written(RelFileLocator locator); +extern void pgstat_report_relfilenode_buffer_read(Relation reln); +extern void pgstat_report_relfilenode_buffer_hit(Relation reln); + /* * If stats are enabled, but pending data hasn't been prepared yet, call * pgstat_assoc_relation() to do so. See its comment for why this is done @@ -685,6 +699,7 @@ extern void pgstat_twophase_postabort(TransactionId xid, uint16 info, void *recdata, uint32 len); extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry(Oid relid); +extern PgStat_StatRelFileNodeEntry *pgstat_fetch_stat_relfilenodeentry(Oid dboid, Oid spcOid, RelFileNumber relfile); extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry_ext(bool shared, Oid reloid); extern PgStat_TableStatus *find_tabstat_entry(Oid rel_id); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index fb132e439d..47c448d5de 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -53,7 +53,8 @@ typedef struct PgStat_HashKey { PgStat_Kind kind; /* statistics entry kind */ Oid dboid; /* database ID. InvalidOid for shared objects. */ - Oid objoid; /* object ID, either table or function. */ + Oid objoid; /* object ID, either table or function or tablespace. */ + RelFileNumber relfile; /* relfilenumber for RelFileLocator. */ } PgStat_HashKey; /* @@ -390,6 +391,12 @@ typedef struct PgStatShared_Relation PgStat_StatTabEntry stats; } PgStatShared_Relation; +typedef struct PgStatShared_RelFileNode +{ + PgStatShared_Common header; + PgStat_StatRelFileNodeEntry stats; +} PgStatShared_RelFileNode; + typedef struct PgStatShared_Function { PgStatShared_Common header; @@ -528,6 +535,9 @@ static inline void *pgstat_get_entry_data(PgStat_Kind kind, PgStatShared_Common static inline void *pgstat_get_custom_shmem_data(PgStat_Kind kind); static inline void *pgstat_get_custom_snapshot_data(PgStat_Kind kind); +extern PgStat_SubXactStatus *pgStatXactStack; +extern void PgStat_RemoveRelFileNodeFromDroppedStats(PgStat_SubXactStatus *xact_state, RelFileLocator rlocator); + /* * Functions in pgstat.c @@ -544,10 +554,12 @@ extern void pgstat_assert_is_up(void); #endif extern void pgstat_delete_pending_entry(PgStat_EntryRef *entry_ref); -extern PgStat_EntryRef *pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, bool *created_entry); -extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid); +extern PgStat_EntryRef *pgstat_prep_pending_entry(PgStat_Kind kind, Oid dboid, + Oid objoid, RelFileNumber relfile, + bool *created_entry); +extern PgStat_EntryRef *pgstat_fetch_pending_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); -extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid); +extern void *pgstat_fetch_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); extern void pgstat_snapshot_fixed(PgStat_Kind kind); @@ -619,6 +631,7 @@ extern void AtPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); extern void PostPrepare_PgStat_Relations(PgStat_SubXactStatus *xact_state); extern bool pgstat_relation_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); +extern bool pgstat_relfilenode_flush_cb(PgStat_EntryRef *entry_ref, bool nowait); extern void pgstat_relation_delete_pending_cb(PgStat_EntryRef *entry_ref); @@ -639,15 +652,16 @@ extern void pgstat_attach_shmem(void); extern void pgstat_detach_shmem(void); extern PgStat_EntryRef *pgstat_get_entry_ref(PgStat_Kind kind, Oid dboid, Oid objoid, - bool create, bool *created_entry); + RelFileNumber relfile, bool create, + bool *created_entry); extern bool pgstat_lock_entry(PgStat_EntryRef *entry_ref, bool nowait); extern bool pgstat_lock_entry_shared(PgStat_EntryRef *entry_ref, bool nowait); extern void pgstat_unlock_entry(PgStat_EntryRef *entry_ref); -extern bool pgstat_drop_entry(PgStat_Kind kind, Oid dboid, Oid objoid); +extern bool pgstat_drop_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); extern void pgstat_drop_all_entries(void); extern PgStat_EntryRef *pgstat_get_entry_ref_locked(PgStat_Kind kind, Oid dboid, Oid objoid, - bool nowait); -extern void pgstat_reset_entry(PgStat_Kind kind, Oid dboid, Oid objoid, TimestampTz ts); + RelFileNumber relfile, bool nowait); +extern void pgstat_reset_entry(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile, TimestampTz ts); extern void pgstat_reset_entries_of_kind(PgStat_Kind kind, TimestampTz ts); extern void pgstat_reset_matching_entries(bool (*do_reset) (PgStatShared_HashEntry *, Datum), Datum match_data, @@ -694,8 +708,8 @@ extern void pgstat_subscription_reset_timestamp_cb(PgStatShared_Common *header, */ extern PgStat_SubXactStatus *pgstat_get_xact_stack_level(int nest_level); -extern void pgstat_drop_transactional(PgStat_Kind kind, Oid dboid, Oid objoid); -extern void pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid); +extern void pgstat_drop_transactional(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); +extern void pgstat_create_transactional(PgStat_Kind kind, Oid dboid, Oid objoid, RelFileNumber relfile); /* diff --git a/src/test/recovery/t/029_stats_restart.pl b/src/test/recovery/t/029_stats_restart.pl index 93a7209f69..f9988b5028 100644 --- a/src/test/recovery/t/029_stats_restart.pl +++ b/src/test/recovery/t/029_stats_restart.pl @@ -40,10 +40,10 @@ trigger_funcrel_stat(); # verify stats objects exist my $sect = "initial"; -is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 't', "$sect: db stats do exist"); +is(have_stats('function', $dboid, $funcoid, 0), 't', "$sect: function stats do exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 't', "$sect: relation stats do exist"); # regular shutdown @@ -64,10 +64,10 @@ copy($og_stats, $statsfile) or die "Copy failed: $!"; $node->start; $sect = "copy"; -is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 't', "$sect: db stats do exist"); +is(have_stats('function', $dboid, $funcoid, 0), 't', "$sect: function stats do exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 't', "$sect: relation stats do exist"); $node->stop('immediate'); @@ -81,10 +81,10 @@ $node->start; # stats should have been discarded $sect = "post immediate"; -is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 'f', "$sect: db stats do not exist"); +is(have_stats('function', $dboid, $funcoid, 0), 'f', "$sect: function stats do exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 'f', "$sect: relation stats do not exist"); # get rid of backup statsfile @@ -95,10 +95,10 @@ unlink $statsfile or die "cannot unlink $statsfile $!"; trigger_funcrel_stat(); $sect = "post immediate, new"; -is(have_stats('database', $dboid, 0), 't', "$sect: db stats do exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 't', "$sect: db stats do exist"); +is(have_stats('function', $dboid, $funcoid, 0), 't', "$sect: function stats do exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 't', "$sect: relation stats do exist"); # regular shutdown @@ -114,10 +114,10 @@ $node->start; # no stats present due to invalid stats file $sect = "invalid_overwrite"; -is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 'f', "$sect: db stats do not exist"); +is(have_stats('function', $dboid, $funcoid, 0), 'f', "$sect: function stats do not exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 'f', "$sect: relation stats do not exist"); @@ -130,10 +130,10 @@ append_file($og_stats, "XYZ"); $node->start; $sect = "invalid_append"; -is(have_stats('database', $dboid, 0), 'f', "$sect: db stats do not exist"); -is(have_stats('function', $dboid, $funcoid), +is(have_stats('database', $dboid, 0, 0), 'f', "$sect: db stats do not exist"); +is(have_stats('function', $dboid, $funcoid, 0), 'f', "$sect: function stats do not exist"); -is(have_stats('relation', $dboid, $tableoid), +is(have_stats('relation', $dboid, $tableoid, 0), 'f', "$sect: relation stats do not exist"); @@ -292,10 +292,10 @@ sub trigger_funcrel_stat sub have_stats { - my ($kind, $dboid, $objoid) = @_; + my ($kind, $dboid, $objoid, $relfile) = @_; return $node->safe_psql($connect_db, - "SELECT pg_stat_have_stats('$kind', $dboid, $objoid)"); + "SELECT pg_stat_have_stats('$kind', $dboid, $objoid, $relfile)"); } sub overwrite_file diff --git a/src/test/recovery/t/030_stats_cleanup_replica.pl b/src/test/recovery/t/030_stats_cleanup_replica.pl index 74b516cc7c..317df24c4f 100644 --- a/src/test/recovery/t/030_stats_cleanup_replica.pl +++ b/src/test/recovery/t/030_stats_cleanup_replica.pl @@ -179,9 +179,9 @@ sub test_standby_func_tab_stats_status my %stats; $stats{rel} = $node_standby->safe_psql($connect_db, - "SELECT pg_stat_have_stats('relation', $dboid, $tableoid)"); + "SELECT pg_stat_have_stats('relation', $dboid, $tableoid, 0)"); $stats{func} = $node_standby->safe_psql($connect_db, - "SELECT pg_stat_have_stats('function', $dboid, $funcoid)"); + "SELECT pg_stat_have_stats('function', $dboid, $funcoid, 0)"); is_deeply(\%stats, \%expected, "$sect: standby stats as expected"); @@ -194,7 +194,7 @@ sub test_standby_db_stats_status my ($connect_db, $dboid, $present) = @_; is( $node_standby->safe_psql( - $connect_db, "SELECT pg_stat_have_stats('database', $dboid, 0)"), + $connect_db, "SELECT pg_stat_have_stats('database', $dboid, 0, 0)"), $present, "$sect: standby db stats as expected"); } diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 5201280669..234356a710 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2335,6 +2335,11 @@ pg_statio_all_tables| SELECT c.oid AS relid, n.nspname AS schemaname, c.relname, (pg_stat_get_blocks_fetched(c.oid) - pg_stat_get_blocks_hit(c.oid)) AS heap_blks_read, + (pg_stat_get_blocks_written(c.oid) + pg_stat_get_relfilenode_blocks_written(d.oid, + CASE + WHEN (c.reltablespace <> (0)::oid) THEN c.reltablespace + ELSE d.dattablespace + END, c.relfilenode)) AS heap_blks_written, pg_stat_get_blocks_hit(c.oid) AS heap_blks_hit, i.idx_blks_read, i.idx_blks_hit, @@ -2342,7 +2347,8 @@ pg_statio_all_tables| SELECT c.oid AS relid, pg_stat_get_blocks_hit(t.oid) AS toast_blks_hit, x.idx_blks_read AS tidx_blks_read, x.idx_blks_hit AS tidx_blks_hit - FROM ((((pg_class c + FROM pg_database d, + ((((pg_class c LEFT JOIN pg_class t ON ((c.reltoastrelid = t.oid))) LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace))) LEFT JOIN LATERAL ( SELECT (sum((pg_stat_get_blocks_fetched(pg_index.indexrelid) - pg_stat_get_blocks_hit(pg_index.indexrelid))))::bigint AS idx_blks_read, @@ -2353,7 +2359,7 @@ pg_statio_all_tables| SELECT c.oid AS relid, (sum(pg_stat_get_blocks_hit(pg_index.indexrelid)))::bigint AS idx_blks_hit FROM pg_index WHERE (pg_index.indrelid = t.oid)) x ON (true)) - WHERE (c.relkind = ANY (ARRAY['r'::"char", 't'::"char", 'm'::"char"])); + WHERE ((c.relkind = ANY (ARRAY['r'::"char", 't'::"char", 'm'::"char"])) AND (d.datname = current_database())); pg_statio_sys_indexes| SELECT relid, indexrelid, schemaname, @@ -2374,6 +2380,7 @@ pg_statio_sys_tables| SELECT relid, schemaname, relname, heap_blks_read, + heap_blks_written, heap_blks_hit, idx_blks_read, idx_blks_hit, @@ -2403,6 +2410,7 @@ pg_statio_user_tables| SELECT relid, schemaname, relname, heap_blks_read, + heap_blks_written, heap_blks_hit, idx_blks_read, idx_blks_hit, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 6e08898b18..eff0c9372c 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1111,23 +1111,23 @@ ROLLBACK; -- pg_stat_have_stats behavior ---- -- fixed-numbered stats exist -SELECT pg_stat_have_stats('bgwriter', 0, 0); +SELECT pg_stat_have_stats('bgwriter', 0, 0, 0); pg_stat_have_stats -------------------- t (1 row) -- unknown stats kinds error out -SELECT pg_stat_have_stats('zaphod', 0, 0); +SELECT pg_stat_have_stats('zaphod', 0, 0, 0); ERROR: invalid statistics kind: "zaphod" -- db stats have objoid 0 -SELECT pg_stat_have_stats('database', :dboid, 1); +SELECT pg_stat_have_stats('database', :dboid, 1, 0); pg_stat_have_stats -------------------- f (1 row) -SELECT pg_stat_have_stats('database', :dboid, 0); +SELECT pg_stat_have_stats('database', :dboid, 0, 0); pg_stat_have_stats -------------------- t @@ -1144,21 +1144,21 @@ select a from stats_test_tab1 where a = 3; 3 (1 row) -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t (1 row) -- pg_stat_have_stats returns false for dropped index with stats -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t (1 row) DROP index stats_test_idx1; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- f @@ -1174,14 +1174,14 @@ select a from stats_test_tab1 where a = 3; 3 (1 row) -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t (1 row) ROLLBACK; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- f @@ -1196,7 +1196,7 @@ select a from stats_test_tab1 where a = 3; 3 (1 row) -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t @@ -1204,7 +1204,7 @@ SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); REINDEX index CONCURRENTLY stats_test_idx1; -- false for previous oid -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- f @@ -1212,7 +1212,7 @@ SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); -- true for new oid SELECT 'stats_test_idx1'::regclass::oid AS stats_test_idx1_oid \gset -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t @@ -1220,7 +1220,7 @@ SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); -- pg_stat_have_stats returns true for a rolled back drop index with stats BEGIN; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t @@ -1228,7 +1228,7 @@ SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); DROP index stats_test_idx1; ROLLBACK; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); pg_stat_have_stats -------------------- t @@ -1513,7 +1513,7 @@ SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_ext (1 row) -- Test IO stats reset -SELECT pg_stat_have_stats('io', 0, 0); +SELECT pg_stat_have_stats('io', 0, 0, 0); pg_stat_have_stats -------------------- t diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index d8ac0d06f4..5a40779989 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -539,12 +539,12 @@ ROLLBACK; -- pg_stat_have_stats behavior ---- -- fixed-numbered stats exist -SELECT pg_stat_have_stats('bgwriter', 0, 0); +SELECT pg_stat_have_stats('bgwriter', 0, 0, 0); -- unknown stats kinds error out -SELECT pg_stat_have_stats('zaphod', 0, 0); +SELECT pg_stat_have_stats('zaphod', 0, 0, 0); -- db stats have objoid 0 -SELECT pg_stat_have_stats('database', :dboid, 1); -SELECT pg_stat_have_stats('database', :dboid, 0); +SELECT pg_stat_have_stats('database', :dboid, 1, 0); +SELECT pg_stat_have_stats('database', :dboid, 0, 0); -- pg_stat_have_stats returns true for committed index creation CREATE table stats_test_tab1 as select generate_series(1,10) a; @@ -552,40 +552,40 @@ CREATE index stats_test_idx1 on stats_test_tab1(a); SELECT 'stats_test_idx1'::regclass::oid AS stats_test_idx1_oid \gset SET enable_seqscan TO off; select a from stats_test_tab1 where a = 3; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- pg_stat_have_stats returns false for dropped index with stats -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); DROP index stats_test_idx1; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- pg_stat_have_stats returns false for rolled back index creation BEGIN; CREATE index stats_test_idx1 on stats_test_tab1(a); SELECT 'stats_test_idx1'::regclass::oid AS stats_test_idx1_oid \gset select a from stats_test_tab1 where a = 3; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); ROLLBACK; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- pg_stat_have_stats returns true for reindex CONCURRENTLY CREATE index stats_test_idx1 on stats_test_tab1(a); SELECT 'stats_test_idx1'::regclass::oid AS stats_test_idx1_oid \gset select a from stats_test_tab1 where a = 3; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); REINDEX index CONCURRENTLY stats_test_idx1; -- false for previous oid -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- true for new oid SELECT 'stats_test_idx1'::regclass::oid AS stats_test_idx1_oid \gset -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- pg_stat_have_stats returns true for a rolled back drop index with stats BEGIN; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); DROP index stats_test_idx1; ROLLBACK; -SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid); +SELECT pg_stat_have_stats('relation', :dboid, :stats_test_idx1_oid, 0); -- put enable_seqscan back to on SET enable_seqscan TO on; @@ -759,7 +759,7 @@ SELECT sum(extends) AS io_sum_bulkwrite_strategy_extends_after SELECT :io_sum_bulkwrite_strategy_extends_after > :io_sum_bulkwrite_strategy_extends_before; -- Test IO stats reset -SELECT pg_stat_have_stats('io', 0, 0); +SELECT pg_stat_have_stats('io', 0, 0, 0); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS io_stats_pre_reset FROM pg_stat_io \gset SELECT pg_stat_reset_shared('io'); diff --git a/src/test/subscription/t/026_stats.pl b/src/test/subscription/t/026_stats.pl index fb3e5629b3..1f4ae5efd5 100644 --- a/src/test/subscription/t/026_stats.pl +++ b/src/test/subscription/t/026_stats.pl @@ -263,7 +263,7 @@ $node_subscriber->safe_psql($db, qq(DROP SUBSCRIPTION $sub1_name)); # Subscription stats for sub1 should be gone is( $node_subscriber->safe_psql( - $db, qq(SELECT pg_stat_have_stats('subscription', 0, $sub1_oid))), + $db, qq(SELECT pg_stat_have_stats('subscription', 0, $sub1_oid, 0))), qq(f), qq(Subscription stats for subscription '$sub1_name' should be removed.)); @@ -282,7 +282,7 @@ DROP SUBSCRIPTION $sub2_name; # Subscription stats for sub2 should be gone is( $node_subscriber->safe_psql( - $db, qq(SELECT pg_stat_have_stats('subscription', 0, $sub2_oid))), + $db, qq(SELECT pg_stat_have_stats('subscription', 0, $sub2_oid, 0))), qq(f), qq(Subscription stats for subscription '$sub2_name' should be removed.)); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 6e6b7c2711..7b6b413c03 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2123,6 +2123,7 @@ PgStatShared_InjectionPoint PgStatShared_InjectionPointFixed PgStatShared_IO PgStatShared_Relation +PgStatShared_RelFileNode PgStatShared_ReplSlot PgStatShared_SLRU PgStatShared_Subscription -- 2.34.1 --fLO+MFZDcAUMk1It-- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ccfbacb4a37..4fa23d992f5 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index fb68f42ce1e..93b041352c5 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 206d4c9fac9..2de610ef729 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -302,8 +302,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -405,6 +410,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v23-0001-Add-CopyToRountine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 74844103228..cd80d34f3da 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -164,7 +164,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -193,7 +192,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -221,7 +220,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -363,7 +362,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -380,7 +379,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -691,14 +690,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -1078,7 +1077,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -2103,8 +2102,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 54aa6cdecaf..cd9e352533a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -63,7 +63,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -99,7 +98,7 @@ CopyToTextLikeSendEndOfRow(CopyToState cstate) } /* Now take the actions related to the end of a row */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -325,7 +324,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -339,7 +338,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -419,8 +418,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -443,8 +442,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 3104d99ea9f..0820b47a2d2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -401,6 +406,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Wed_Jul_24_17_30_59_2024_070)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ec86a17b3b3..64772877b0f 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 37b150b44ba..c99edae575b 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 8a560903ede..c1e9fe366f3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -402,6 +407,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v21-0001-Add-CopyToRountine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 74844103228..a115d7f9e26 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 54aa6cdecaf..b8d0e996117 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -500,6 +500,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index e298b19860c..5665408eaa0 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -402,6 +407,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Jul_30_16_13_06_2024_473)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). --- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cfc74ee7b1..b5d8678394 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) default: break; } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } CopyToRoutine CopyToRoutineText = { @@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index a869d78d72..ffad433a21 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -174,6 +174,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.43.0 ----Next_Part(Wed_Jan_24_14_49_36_2024_411)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v6-0005-Extract-COPY-FROM-format-implementations.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension @ 2024-01-24 05:19 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-24 05:19 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index a78a790060..f8a194635d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b7e8f627bf..22accc83ab 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -314,8 +314,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + /* * Represents the different dest cases we need to worry about at * the bottom level -- 2.43.0 ----Next_Part(Wed_Jan_24_14_49_36_2024_411)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension @ 2024-09-28 14:59 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 37b150b44ba..c99edae575b 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 03779c15f43..30765951e2e 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -187,6 +187,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v21-0006-Add-CopyFromRoutine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension @ 2024-09-28 14:59 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index fb68f42ce1e..93b041352c5 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b6ddb5f6216..310a37ba728 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -190,6 +190,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v23-0006-Add-CopyFromRoutine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension @ 2024-09-28 15:32 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ccfbacb4a37..4fa23d992f5 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 0274e3487c3..2de610ef729 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -302,8 +302,14 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + + typedef struct CopyToStateData *CopyToState; /* -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension @ 2024-09-28 15:32 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ec86a17b3b3..64772877b0f 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c118558ee71..c1e9fe366f3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,14 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + + typedef struct CopyToStateData *CopyToState; /* -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 96b5e144a1d..cb9bfa0053f 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.45.2 ----Next_Part(Wed_Nov_27_16_53_44_2024_871)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v27-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 91fa46ddf6f..da281f32950 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 5d071b378d6..f8167af4c79 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 2df53dda8a0..4b82372691e 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts); -- 2.47.1 ----Next_Part(Sat_Feb__1_08_10_23_2025_403)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v30-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 17d89c23af0..35f9035141a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Sat_Mar__1_11_50_09_2025_878)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v35-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 91fa46ddf6f..da281f32950 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 5d071b378d6..f8167af4c79 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 2df53dda8a0..4b82372691e 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts); -- 2.47.1 ----Next_Part(Fri_Jan_31_00_42_13_2025_303)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v29-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Wed_Mar_19_11_56_17_2025_532)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v37-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 1e75e07dc0b..7487190bdd6 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.47.1 ----Next_Part(Thu_Jan_23_18_12_10_2025_763)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v28-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 16d3b389e97..20d49d73e38 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c125dc3e209..d0da9e07a0d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.1 ----Next_Part(Thu_Feb__6_21_06_31_2025_538)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v32-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Wed_Mar__5_09_06_08_2025_350)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v36-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 16d3b389e97..20d49d73e38 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c125dc3e209..d0da9e07a0d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.1 ----Next_Part(Sat_Feb__1_19_12_01_2025_760)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v31-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 96b5e144a1d..cb9bfa0053f 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.45.2 ----Next_Part(Mon_Nov_25_15_01_50_2024_156)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v26-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Thu_Mar_20_10_24_55_2025_309)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v38-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 5fcdbea2c2a..d79b6ebe8a3 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 9358515c6f6..6f158272829 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.47.1 ----Next_Part(Thu_Jan_23_18_12_10_2025_763)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v28-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Wed_Mar__5_09_06_08_2025_350)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v36-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 1c68b0d2952..0a7e7255b7d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 335584f8877..caba308533d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Wed_Nov_27_16_53_44_2024_871)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v27-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Sat_Mar__1_11_50_09_2025_878)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v35-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 1c68b0d2952..0a7e7255b7d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 335584f8877..caba308533d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Mon_Nov_25_15_01_50_2024_156)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 75b49629f08..01f2e7a8824 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 103eb21767d..ac58adbd23d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -104,4 +104,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Thu_Feb__6_21_06_31_2025_538)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v32-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index f7982bf692f..650b6b2382b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index bf933069fea..d1a1dbeb178 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -105,4 +105,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3743b11faa4..a65bbbc962e 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Sat_Feb__1_08_10_23_2025_403)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v30-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index f7982bf692f..650b6b2382b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index bf933069fea..d1a1dbeb178 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -105,4 +105,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3743b11faa4..a65bbbc962e 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Fri_Jan_31_00_42_13_2025_303)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v29-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 75b49629f08..01f2e7a8824 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 103eb21767d..ac58adbd23d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -104,4 +104,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Sat_Feb__1_19_12_01_2025_760)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v31-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Wed_Mar_19_11_56_17_2025_532)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v37-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Thu_Mar_20_10_24_55_2025_309)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v38-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v39 3/5] Add support for implementing custom COPY handler as extension @ 2025-03-27 02:24 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw) * TO: Add CopyToStateData::opaque that can be used to keep data for custom COPY TO handler implementation * TO: Export CopySendEndOfRow() to send end of row data as CopyToStateFlush() * FROM: Add CopyFromStateData::opaque that can be used to keep data for custom COPY FROM handler implementation * FROM: Export CopyGetData() to get the next data as CopyFromStateGetData() * FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and "LOG_VERBOSITY verbose" COPY FROM extensions must call CopyFromSkipErrorRow() when CopyFromOneRow callback reports an error by errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases. --- src/backend/commands/copyfromparse.c | 93 ++++++++++++------- src/backend/commands/copyto.c | 12 +++ src/include/commands/copyapi.h | 6 ++ src/include/commands/copyfrom_internal.h | 3 + src/include/commands/copyto_internal.h | 3 + .../test_copy_format/expected/no_schema.out | 47 ++++++++++ .../test_copy_format/sql/no_schema.sql | 24 +++++ .../test_copy_format/test_copy_format.c | 80 +++++++++++++++- 8 files changed, 231 insertions(+), 37 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..2070f51a963 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. @@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true); } +/* + * Call this when you report an error by errsave() in your CopyFromOneRow + * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases + * for you. + */ +void +CopyFromSkipErrorRow(CopyFromState cstate) +{ + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + + cstate->num_errors++; + + if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) + { + /* + * Since we emit line number and column info in the below notice + * message, we suppress error context information other than the + * relation name. + */ + Assert(!cstate->relname_only); + cstate->relname_only = true; + + if (cstate->cur_attval) + { + char *attval; + + attval = CopyLimitPrintoutLength(cstate->cur_attval); + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname, + attval)); + pfree(attval); + } + else + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname)); + + /* reset relname_only */ + cstate->relname_only = false; + } +} + /* * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow(). * @@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { - Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); - - cstate->num_errors++; - - if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) - { - /* - * Since we emit line number and column info in the below - * notice message, we suppress error context information other - * than the relation name. - */ - Assert(!cstate->relname_only); - cstate->relname_only = true; - - if (cstate->cur_attval) - { - char *attval; - - attval = CopyLimitPrintoutLength(cstate->cur_attval); - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"", - (unsigned long long) cstate->cur_lineno, - cstate->cur_attname, - attval)); - pfree(attval); - } - else - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input", - (unsigned long long) cstate->cur_lineno, - cstate->cur_attname)); - - /* reset relname_only */ - cstate->relname_only = false; - } - + CopyFromSkipErrorRow(cstate); return true; } diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b7ff6466ce3..23cbdad184c 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -456,6 +456,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 53ad3337f86..500ece7d5bb 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. @@ -106,4 +108,8 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + +extern void CopyFromSkipErrorRow(CopyFromState cstate); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 12c4a0f5979..14ee0f50588 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ diff --git a/src/test/modules/test_copy_format/expected/no_schema.out b/src/test/modules/test_copy_format/expected/no_schema.out index d5903632b2e..05d160c1eae 100644 --- a/src/test/modules/test_copy_format/expected/no_schema.out +++ b/src/test/modules/test_copy_format/expected/no_schema.out @@ -1,6 +1,8 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=true NOTICE: CopyFromInFunc: attribute: smallint @@ -8,7 +10,50 @@ NOTICE: CopyFromInFunc: attribute: integer NOTICE: CopyFromInFunc: attribute: bigint NOTICE: CopyFromStart: the number of attributes: 3 NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: invalid value: "6" +CONTEXT: COPY test, line 2, column a: "6" +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: skipping row due to data type incompatibility at line 2 for column "a": "6" +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility +NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: too much lines: 3 +CONTEXT: COPY test, line 3 COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=false NOTICE: CopyToOutFunc: attribute: smallint @@ -18,6 +63,8 @@ NOTICE: CopyToStart: the number of attributes: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToEnd DROP TABLE public.test; DROP EXTENSION test_copy_format; diff --git a/src/test/modules/test_copy_format/sql/no_schema.sql b/src/test/modules/test_copy_format/sql/no_schema.sql index 1e049f799f0..1901c4a9f43 100644 --- a/src/test/modules/test_copy_format/sql/no_schema.sql +++ b/src/test/modules/test_copy_format/sql/no_schema.sql @@ -1,7 +1,31 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +321 \. COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); DROP TABLE public.test; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 1d754201336..34ec693a7ec 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "commands/copyapi.h" +#include "commands/copyfrom_internal.h" #include "commands/defrem.h" #include "utils/builtins.h" @@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) static bool TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { + int n_attributes = list_length(cstate->attnumlist); + char *line; + int line_size = n_attributes + 1; /* +1 is for new line */ + int read_bytes; + ereport(NOTICE, (errmsg("CopyFromOneRow"))); - return false; + + cstate->cur_lineno++; + line = palloc(line_size); + read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size); + if (read_bytes == 0) + return false; + if (read_bytes != line_size) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("one line must be %d bytes: %d", + line_size, read_bytes))); + + if (cstate->cur_lineno == 1) + { + /* Success */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + ListCell *cur; + int i = 0; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (att->atttypid == INT2OID) + { + values[i] = Int16GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT4OID) + { + values[i] = Int32GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT8OID) + { + values[i] = Int64GetDatum(line[i] - '0'); + } + nulls[i] = false; + i++; + } + } + else if (cstate->cur_lineno == 2) + { + /* Soft error */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + int attnum = lfirst_int(list_head(cstate->attnumlist)); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + char value[2]; + + cstate->cur_attname = NameStr(att->attname); + value[0] = line[0]; + value[1] = '\0'; + cstate->cur_attval = value; + errsave((Node *) cstate->escontext, + ( + errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid value: \"%c\"", line[0]))); + CopyFromSkipErrorRow(cstate); + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; + return true; + } + else + { + /* Hard error */ + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("too much lines: %llu", + (unsigned long long) cstate->cur_lineno))); + } + + return true; } static void -- 2.47.2 ----Next_Part(Thu_Mar_27_12_28_40_2025_510)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v39-0004-Use-copy-handlers-for-built-in-formats.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v40 3/6] Add support for implementing custom COPY handler as extension @ 2025-03-27 02:24 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw) * TO: Add CopyToStateData::opaque that can be used to keep data for custom COPY TO handler implementation * TO: Export CopySendEndOfRow() to send end of row data as CopyToStateFlush() * FROM: Add CopyFromStateData::opaque that can be used to keep data for custom COPY FROM handler implementation * FROM: Export CopyGetData() to get the next data as CopyFromStateGetData() * FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and "LOG_VERBOSITY verbose" COPY FROM extensions must call CopyFromSkipErrorRow() when CopyFromOneRow callback reports an error by errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases. --- src/backend/commands/copyfromparse.c | 93 ++++++++++++------- src/backend/commands/copyto.c | 12 +++ src/include/commands/copyapi.h | 6 ++ src/include/commands/copyfrom_internal.h | 3 + src/include/commands/copyto_internal.h | 3 + .../expected/test_copy_format.out | 50 ++++++++++ .../test_copy_format/sql/test_copy_format.sql | 35 +++++++ .../test_copy_format/test_copy_format.c | 80 +++++++++++++++- 8 files changed, 245 insertions(+), 37 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 9f7171d1478..de68b53b000 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. @@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true); } +/* + * Call this when you report an error by errsave() in your CopyFromOneRow + * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases + * for you. + */ +void +CopyFromSkipErrorRow(CopyFromState cstate) +{ + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + + cstate->num_errors++; + + if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) + { + /* + * Since we emit line number and column info in the below notice + * message, we suppress error context information other than the + * relation name. + */ + Assert(!cstate->relname_only); + cstate->relname_only = true; + + if (cstate->cur_attval) + { + char *attval; + + attval = CopyLimitPrintoutLength(cstate->cur_attval); + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", + cstate->cur_lineno, + cstate->cur_attname, + attval)); + pfree(attval); + } + else + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", + cstate->cur_lineno, + cstate->cur_attname)); + + /* reset relname_only */ + cstate->relname_only = false; + } +} + /* * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow(). * @@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { - Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); - - cstate->num_errors++; - - if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) - { - /* - * Since we emit line number and column info in the below - * notice message, we suppress error context information other - * than the relation name. - */ - Assert(!cstate->relname_only); - cstate->relname_only = true; - - if (cstate->cur_attval) - { - char *attval; - - attval = CopyLimitPrintoutLength(cstate->cur_attval); - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", - cstate->cur_lineno, - cstate->cur_attname, - attval)); - pfree(attval); - } - else - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", - cstate->cur_lineno, - cstate->cur_attname)); - - /* reset relname_only */ - cstate->relname_only = false; - } - + CopyFromSkipErrorRow(cstate); return true; } diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 265b847e255..d6fcfdfb9b1 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -454,6 +454,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 53ad3337f86..500ece7d5bb 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. @@ -106,4 +108,8 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + +extern void CopyFromSkipErrorRow(CopyFromState cstate); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 24157e11a73..f9e27152313 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index da796131988..3bd9d702bf0 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out index 3916b766615..47a875f0ab1 100644 --- a/src/test/modules/test_copy_format/expected/test_copy_format.out +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -4,6 +4,8 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); -- schema. CREATE EXTENSION test_copy_format; -- We can find a custom COPY handler without schema. +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=true NOTICE: CopyFromInFunc: attribute: smallint @@ -11,7 +13,50 @@ NOTICE: CopyFromInFunc: attribute: integer NOTICE: CopyFromInFunc: attribute: bigint NOTICE: CopyFromStart: the number of attributes: 3 NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: invalid value: "6" +CONTEXT: COPY copy_data, line 2, column a: "6" +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: skipping row due to data type incompatibility at line 2 for column "a": "6" +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility +NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: too much lines: 3 +CONTEXT: COPY copy_data, line 3 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=false NOTICE: CopyToOutFunc: attribute: smallint @@ -21,7 +66,12 @@ NOTICE: CopyToStart: the number of attributes: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToEnd +-- Reset data. +TRUNCATE copy_data; +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); DROP EXTENSION test_copy_format; -- Install custom COPY handlers to a schema that isn't included in -- search_path. diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql index b262794f878..c7beb2fb8ae 100644 --- a/src/test/modules/test_copy_format/sql/test_copy_format.sql +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -4,10 +4,45 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); -- No WITH SCHEMA. It installs custom COPY handlers to the current -- schema. CREATE EXTENSION test_copy_format; + -- We can find a custom COPY handler without schema. + +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format'); +987 +654 \. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +\. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +987 +654 +\. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +321 +\. + COPY copy_data TO stdout WITH (FORMAT 'test_copy_format'); + +-- Reset data. +TRUNCATE copy_data; +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); + DROP EXTENSION test_copy_format; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 1d754201336..34ec693a7ec 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "commands/copyapi.h" +#include "commands/copyfrom_internal.h" #include "commands/defrem.h" #include "utils/builtins.h" @@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) static bool TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { + int n_attributes = list_length(cstate->attnumlist); + char *line; + int line_size = n_attributes + 1; /* +1 is for new line */ + int read_bytes; + ereport(NOTICE, (errmsg("CopyFromOneRow"))); - return false; + + cstate->cur_lineno++; + line = palloc(line_size); + read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size); + if (read_bytes == 0) + return false; + if (read_bytes != line_size) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("one line must be %d bytes: %d", + line_size, read_bytes))); + + if (cstate->cur_lineno == 1) + { + /* Success */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + ListCell *cur; + int i = 0; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (att->atttypid == INT2OID) + { + values[i] = Int16GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT4OID) + { + values[i] = Int32GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT8OID) + { + values[i] = Int64GetDatum(line[i] - '0'); + } + nulls[i] = false; + i++; + } + } + else if (cstate->cur_lineno == 2) + { + /* Soft error */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + int attnum = lfirst_int(list_head(cstate->attnumlist)); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + char value[2]; + + cstate->cur_attname = NameStr(att->attname); + value[0] = line[0]; + value[1] = '\0'; + cstate->cur_attval = value; + errsave((Node *) cstate->escontext, + ( + errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid value: \"%c\"", line[0]))); + CopyFromSkipErrorRow(cstate); + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; + return true; + } + else + { + /* Hard error */ + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("too much lines: %llu", + (unsigned long long) cstate->cur_lineno))); + } + + return true; } static void -- 2.47.2 ----Next_Part(Fri_Apr_25_21_45_34_2025_836)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v40-0004-Use-copy-handlers-for-built-in-formats.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
end of thread, other threads:[~2025-03-27 02:24 UTC | newest] Thread overview: 35+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2023-11-16 02:30 [PATCH v3] Provide relfilenode statistics Bertrand Drouvot <[email protected]> 2024-01-23 06:12 [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-01-24 05:19 [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-09-28 14:59 [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-09-28 14:59 [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-09-28 15:32 [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-09-28 15:32 [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2025-03-27 02:24 [PATCH v39 3/5] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]> 2025-03-27 02:24 [PATCH v40 3/6] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox