agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v6 2/3] Make resowners more easily extensible. 35+ messages / 2 participants [nested] [flat]
* [PATCH v6 2/3] Make resowners more easily extensible. @ 2021-01-20 21:16 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Heikki Linnakangas @ 2021-01-20 21:16 UTC (permalink / raw) Use a single array and hash, instead of one for each object kind. --- src/backend/access/common/tupdesc.c | 42 +- src/backend/jit/jit.c | 2 - src/backend/jit/llvm/llvmjit.c | 46 +- src/backend/storage/buffer/bufmgr.c | 45 +- src/backend/storage/buffer/localbuf.c | 4 +- src/backend/storage/file/fd.c | 44 +- src/backend/storage/ipc/dsm.c | 44 +- src/backend/storage/lmgr/lock.c | 2 +- src/backend/utils/cache/catcache.c | 74 +- src/backend/utils/cache/plancache.c | 45 +- src/backend/utils/cache/relcache.c | 41 +- src/backend/utils/resowner/README | 19 +- src/backend/utils/resowner/resowner.c | 1236 +++++++------------------ src/backend/utils/time/snapmgr.c | 38 +- src/common/cryptohash_openssl.c | 49 +- src/include/storage/buf_internals.h | 9 + src/include/utils/catcache.h | 3 - src/include/utils/plancache.h | 2 + src/include/utils/resowner.h | 45 +- src/include/utils/resowner_private.h | 105 --- src/tools/pgindent/typedefs.list | 4 +- 21 files changed, 844 insertions(+), 1055 deletions(-) delete mode 100644 src/include/utils/resowner_private.h diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 902f59440cd..2bf0c121fda 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -29,9 +29,27 @@ #include "utils/acl.h" #include "utils/builtins.h" #include "utils/datum.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/syscache.h" +/* ResourceOwner callbacks to hold tupledesc references */ +static void ResOwnerReleaseTupleDesc(Datum res); +static void ResOwnerPrintTupleDescLeakWarning(Datum res); + +static ResourceOwnerFuncs tupdesc_resowner_funcs = +{ + /* relcache references */ + .name = "tupdesc reference", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseTupleDesc, + .PrintLeakWarning = ResOwnerPrintTupleDescLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberTupleDesc(owner, tupdesc) \ + ResourceOwnerRemember(owner, PointerGetDatum(tupdesc), &tupdesc_resowner_funcs) +#define ResourceOwnerForgetTupleDesc(owner, tupdesc) \ + ResourceOwnerForget(owner, PointerGetDatum(tupdesc), &tupdesc_resowner_funcs) /* * CreateTemplateTupleDesc @@ -376,7 +394,7 @@ IncrTupleDescRefCount(TupleDesc tupdesc) { Assert(tupdesc->tdrefcount >= 0); - ResourceOwnerEnlargeTupleDescs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); tupdesc->tdrefcount++; ResourceOwnerRememberTupleDesc(CurrentResourceOwner, tupdesc); } @@ -925,3 +943,23 @@ BuildDescFromLists(List *names, List *types, List *typmods, List *collations) return desc; } + + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseTupleDesc(Datum res) +{ + DecrTupleDescRefCount((TupleDesc) DatumGetPointer(res)); +} + +static void +ResOwnerPrintTupleDescLeakWarning(Datum res) +{ + TupleDesc tupdesc = (TupleDesc) DatumGetPointer(res); + + elog(WARNING, + "TupleDesc reference leak: TupleDesc %p (%u,%d) still referenced", + tupdesc, tupdesc->tdtypeid, tupdesc->tdtypmod); +} diff --git a/src/backend/jit/jit.c b/src/backend/jit/jit.c index 2da300e000d..1aa04d173b4 100644 --- a/src/backend/jit/jit.c +++ b/src/backend/jit/jit.c @@ -26,7 +26,6 @@ #include "jit/jit.h" #include "miscadmin.h" #include "utils/fmgrprotos.h" -#include "utils/resowner_private.h" /* GUCs */ bool jit_enabled = true; @@ -140,7 +139,6 @@ jit_release_context(JitContext *context) if (provider_successfully_loaded) provider.release_context(context); - ResourceOwnerForgetJIT(context->resowner, PointerGetDatum(context)); pfree(context); } diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c index b0789a5fb80..8e5b86a0e5b 100644 --- a/src/backend/jit/llvm/llvmjit.c +++ b/src/backend/jit/llvm/llvmjit.c @@ -40,7 +40,7 @@ #include "portability/instr_time.h" #include "storage/ipc.h" #include "utils/memutils.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" /* Handle of a module emitted via ORC JIT */ typedef struct LLVMJitHandle @@ -121,8 +121,26 @@ static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm); static char *llvm_error_message(LLVMErrorRef error); #endif /* LLVM_VERSION_MAJOR > 11 */ -PG_MODULE_MAGIC; +/* ResourceOwner callbacks to hold JitContexts */ +static void ResOwnerReleaseJitContext(Datum res); +static void ResOwnerPrintJitContextLeakWarning(Datum res); + +static ResourceOwnerFuncs jit_resowner_funcs = +{ + /* relcache references */ + .name = "LLVM JIT context", + .phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .ReleaseResource = ResOwnerReleaseJitContext, + .PrintLeakWarning = ResOwnerPrintJitContextLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberJIT(owner, handle) \ + ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_funcs) +#define ResourceOwnerForgetJIT(owner, handle) \ + ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_funcs) +PG_MODULE_MAGIC; /* * Initialize LLVM JIT provider. @@ -151,7 +169,7 @@ llvm_create_context(int jitFlags) llvm_session_initialize(); - ResourceOwnerEnlargeJIT(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); context = MemoryContextAllocZero(TopMemoryContext, sizeof(LLVMJitContext)); @@ -159,7 +177,7 @@ llvm_create_context(int jitFlags) /* ensure cleanup */ context->base.resowner = CurrentResourceOwner; - ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context)); + ResourceOwnerRememberJIT(CurrentResourceOwner, context); return context; } @@ -221,6 +239,8 @@ llvm_release_context(JitContext *context) pfree(jit_handle); } + + ResourceOwnerForgetJIT(context->resowner, context); } /* @@ -1231,3 +1251,21 @@ llvm_error_message(LLVMErrorRef error) } #endif /* LLVM_VERSION_MAJOR > 11 */ + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseJitContext(Datum res) +{ + jit_release_context((JitContext *) PointerGetDatum(res)); +} + +static void +ResOwnerPrintJitContextLeakWarning(Datum res) +{ + /* XXX: We used to not print these. Was that intentional? */ + JitContext *context = (JitContext *) PointerGetDatum(res); + + elog(WARNING, "JIT context leak: context %p still referenced", context); +} diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index cde869e7d64..82e39de19f9 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -52,7 +52,7 @@ #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/timestamp.h" @@ -206,6 +206,18 @@ static PrivateRefCountEntry *GetPrivateRefCountEntry(Buffer buffer, bool do_move static inline int32 GetPrivateRefCount(Buffer buffer); static void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref); +/* ResourceOwner callbacks to hold buffer pins */ +static void ResOwnerReleaseBuffer(Datum res); +static void ResOwnerPrintBufferLeakWarning(Datum res); + +ResourceOwnerFuncs buffer_resowner_funcs = +{ + .name = "buffer", + .phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .ReleaseResource = ResOwnerReleaseBuffer, + .PrintLeakWarning = ResOwnerPrintBufferLeakWarning +}; + /* * Ensure that the PrivateRefCountArray has sufficient space to store one more * entry. This has to be called before using NewPrivateRefCountEntry() to fill @@ -1094,7 +1106,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * pin. */ ReservePrivateRefCountEntry(); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* * Select a victim buffer. The buffer is returned with its header @@ -1604,7 +1616,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) bool result; PrivateRefCountEntry *ref; - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); ref = GetPrivateRefCountEntry(b, true); @@ -1687,7 +1699,7 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy) * * As this function is called with the spinlock held, the caller has to * previously call ReservePrivateRefCountEntry() and - * ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + * ResourceOwnerEnlarge(CurrentResourceOwner); * * Currently, no callers of this function want to modify the buffer's * usage_count at all, so there's no need for a strategy parameter. @@ -2423,7 +2435,7 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) /* Make sure we can handle the pin */ ReservePrivateRefCountEntry(); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* * Check whether buffer needs writing. @@ -3496,7 +3508,7 @@ FlushRelationBuffers(Relation rel) /* Make sure we can handle the pin */ ReservePrivateRefCountEntry(); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); buf_state = LockBufHdr(bufHdr); if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) && @@ -3591,7 +3603,7 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) /* Make sure we can handle the pin */ ReservePrivateRefCountEntry(); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); buf_state = LockBufHdr(bufHdr); if (RelFileNodeEquals(bufHdr->tag.rnode, srelent->rnode) && @@ -3646,7 +3658,7 @@ FlushDatabaseBuffers(Oid dbid) /* Make sure we can handle the pin */ ReservePrivateRefCountEntry(); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); buf_state = LockBufHdr(bufHdr); if (bufHdr->tag.rnode.dbNode == dbid && @@ -3729,7 +3741,7 @@ void IncrBufferRefCount(Buffer buffer) { Assert(BufferIsPinned(buffer)); - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); if (BufferIsLocal(buffer)) LocalRefCount[-buffer - 1]++; else @@ -4803,3 +4815,18 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) (errcode(ERRCODE_SNAPSHOT_TOO_OLD), errmsg("snapshot too old"))); } + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseBuffer(Datum res) +{ + ReleaseBuffer(DatumGetInt32(res)); +} + +static void +ResOwnerPrintBufferLeakWarning(Datum res) +{ + PrintBufferLeakWarning(DatumGetInt32(res)); +} diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index f7c15ea8a44..3f22d7127b9 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -22,7 +22,7 @@ #include "storage/bufmgr.h" #include "utils/guc.h" #include "utils/memutils.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" /*#define LBDEBUG*/ @@ -124,7 +124,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, InitLocalBuffers(); /* Make sure we will have room to remember the buffer pin */ - ResourceOwnerEnlargeBuffers(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* See if the desired buffer already exists */ hresult = (LocalBufferLookupEnt *) diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index b58502837aa..1172ea245e0 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -97,7 +97,7 @@ #include "storage/fd.h" #include "storage/ipc.h" #include "utils/guc.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" /* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */ #if defined(HAVE_SYNC_FILE_RANGE) @@ -340,6 +340,24 @@ static void unlink_if_exists_fname(const char *fname, bool isdir, int elevel); static int fsync_parent_path(const char *fname, int elevel); +/* ResourceOwner callbacks to hold virtual file descriptors */ +static void ResOwnerReleaseFile(Datum res); +static void ResOwnerPrintFileLeakWarning(Datum res); + +static ResourceOwnerFuncs file_resowner_funcs = +{ + .name = "File", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseFile, + .PrintLeakWarning = ResOwnerPrintFileLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberFile(owner, file) \ + ResourceOwnerRemember(owner, Int32GetDatum(file), &file_resowner_funcs) +#define ResourceOwnerForgetFile(owner, file) \ + ResourceOwnerForget(owner, Int32GetDatum(file), &file_resowner_funcs) + /* * pg_fsync --- do fsync with or without writethrough */ @@ -1430,7 +1448,7 @@ ReportTemporaryFileUsage(const char *path, off_t size) /* * Called to register a temporary file for automatic close. - * ResourceOwnerEnlargeFiles(CurrentResourceOwner) must have been called + * ResourceOwnerEnlarge(CurrentResourceOwner) must have been called * before the file was opened. */ static void @@ -1612,7 +1630,7 @@ OpenTemporaryFile(bool interXact) * open it, if we'll be registering it below. */ if (!interXact) - ResourceOwnerEnlargeFiles(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* * If some temp tablespace(s) have been given to us, try to use the next @@ -1742,7 +1760,7 @@ PathNameCreateTemporaryFile(const char *path, bool error_on_failure) { File file; - ResourceOwnerEnlargeFiles(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* * Open the file. Note: we don't use O_EXCL, in case there is an orphaned @@ -1780,7 +1798,7 @@ PathNameOpenTemporaryFile(const char *path, int mode) { File file; - ResourceOwnerEnlargeFiles(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); file = PathNameOpenFile(path, mode | PG_BINARY); @@ -3700,3 +3718,19 @@ pg_pwritev_with_retry(int fd, const struct iovec *iov, int iovcnt, off_t offset) return sum; } + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseFile(Datum res) +{ + FileClose((File) DatumGetInt32(res)); +} + +static void +ResOwnerPrintFileLeakWarning(Datum res) +{ + elog(WARNING, "temporary file leak: File %d still referenced", + DatumGetInt32(res)); +} diff --git a/src/backend/storage/ipc/dsm.c b/src/backend/storage/ipc/dsm.c index ae82b4bdc0e..31e8860fb53 100644 --- a/src/backend/storage/ipc/dsm.c +++ b/src/backend/storage/ipc/dsm.c @@ -37,13 +37,15 @@ #include "miscadmin.h" #include "port/pg_bitutils.h" #include "storage/dsm.h" +#include "storage/fd.h" #include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/pg_shmem.h" +#include "storage/shmem.h" #include "utils/freepage.h" #include "utils/guc.h" #include "utils/memutils.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #define PG_DYNSHMEM_CONTROL_MAGIC 0x9a503d32 @@ -139,6 +141,25 @@ static dsm_control_header *dsm_control; static Size dsm_control_mapped_size = 0; static void *dsm_control_impl_private = NULL; + +/* ResourceOwner callbacks to hold DSM segments */ +static void ResOwnerReleaseDSM(Datum res); +static void ResOwnerPrintDSMLeakWarning(Datum res); + +static ResourceOwnerFuncs dsm_resowner_funcs = +{ + .name = "dynamic shared memory segment", + .phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .ReleaseResource = ResOwnerReleaseDSM, + .PrintLeakWarning = ResOwnerPrintDSMLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberDSM(owner, seg) \ + ResourceOwnerRemember(owner, PointerGetDatum(seg), &dsm_resowner_funcs) +#define ResourceOwnerForgetDSM(owner, seg) \ + ResourceOwnerForget(owner, PointerGetDatum(seg), &dsm_resowner_funcs) + /* * Start up the dynamic shared memory system. * @@ -895,7 +916,7 @@ void dsm_unpin_mapping(dsm_segment *seg) { Assert(seg->resowner == NULL); - ResourceOwnerEnlargeDSMs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); seg->resowner = CurrentResourceOwner; ResourceOwnerRememberDSM(seg->resowner, seg); } @@ -1162,7 +1183,7 @@ dsm_create_descriptor(void) dsm_segment *seg; if (CurrentResourceOwner) - ResourceOwnerEnlargeDSMs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); seg = MemoryContextAlloc(TopMemoryContext, sizeof(dsm_segment)); dlist_push_head(&dsm_segment_list, &seg->node); @@ -1241,3 +1262,20 @@ is_main_region_dsm_handle(dsm_handle handle) { return handle & 1; } + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseDSM(Datum res) +{ + dsm_detach((dsm_segment *) DatumGetPointer(res)); +} +static void +ResOwnerPrintDSMLeakWarning(Datum res) +{ + dsm_segment *seg = (dsm_segment *) res; + + elog(WARNING, "dynamic shared memory leak: segment %u still referenced", + dsm_segment_handle(seg)); +} diff --git a/src/backend/storage/lmgr/lock.c b/src/backend/storage/lmgr/lock.c index 20e50247ea4..de39c844d49 100644 --- a/src/backend/storage/lmgr/lock.c +++ b/src/backend/storage/lmgr/lock.c @@ -47,7 +47,7 @@ #include "storage/standby.h" #include "utils/memutils.h" #include "utils/ps_status.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" /* This configuration variable is used to set the lock table size */ diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 86fdfc454d2..9f9db426274 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -31,12 +31,13 @@ #endif #include "storage/lmgr.h" #include "utils/builtins.h" +#include "utils/catcache.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/memutils.h" #include "utils/rel.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/syscache.h" @@ -104,6 +105,42 @@ static void CatCacheCopyKeys(TupleDesc tupdesc, int nkeys, int *attnos, * internal support functions */ +/* ResourceOwner callbacks to hold catcache references */ + +static void ResOwnerReleaseCatCache(Datum res); +static void ResOwnerPrintCatCacheLeakWarning(Datum res); +static void ResOwnerReleaseCatCacheList(Datum res); +static void ResOwnerPrintCatCacheListLeakWarning(Datum res); + +static ResourceOwnerFuncs catcache_resowner_funcs = +{ + /* catcache references */ + .name = "catcache reference", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseCatCache, + .PrintLeakWarning = ResOwnerPrintCatCacheLeakWarning +}; + +static ResourceOwnerFuncs catlistref_resowner_funcs = +{ + /* catcache-list pins */ + .name = "catcache list reference", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseCatCacheList, + .PrintLeakWarning = ResOwnerPrintCatCacheListLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberCatCacheRef(owner, tuple) \ + ResourceOwnerRemember(owner, PointerGetDatum(tuple), &catcache_resowner_funcs) +#define ResourceOwnerForgetCatCacheRef(owner, tuple) \ + ResourceOwnerForget(owner, PointerGetDatum(tuple), &catcache_resowner_funcs) +#define ResourceOwnerRememberCatCacheListRef(owner, list) \ + ResourceOwnerRemember(owner, PointerGetDatum(list), &catlistref_resowner_funcs) +#define ResourceOwnerForgetCatCacheListRef(owner, list) \ + ResourceOwnerForget(owner, PointerGetDatum(list), &catlistref_resowner_funcs) + + /* * Hash and equality functions for system types that are used as cache key * fields. In some cases, we just call the regular SQL-callable functions for @@ -1270,7 +1307,7 @@ SearchCatCacheInternal(CatCache *cache, */ if (!ct->negative) { - ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); ct->refcount++; ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple); @@ -1371,7 +1408,7 @@ SearchCatCacheMiss(CatCache *cache, hashValue, hashIndex, false); /* immediately set the refcount to 1 */ - ResourceOwnerEnlargeCatCacheRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); ct->refcount++; ResourceOwnerRememberCatCacheRef(CurrentResourceOwner, &ct->tuple); break; /* assume only one match */ @@ -1583,7 +1620,7 @@ SearchCatCacheList(CatCache *cache, dlist_move_head(&cache->cc_lists, &cl->cache_elem); /* Bump the list's refcount and return it */ - ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); cl->refcount++; ResourceOwnerRememberCatCacheListRef(CurrentResourceOwner, cl); @@ -1695,7 +1732,7 @@ SearchCatCacheList(CatCache *cache, table_close(relation, AccessShareLock); /* Make sure the resource owner has room to remember this entry. */ - ResourceOwnerEnlargeCatCacheListRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); /* Now we can build the CatCList entry. */ oldcxt = MemoryContextSwitchTo(CacheMemoryContext); @@ -2069,14 +2106,19 @@ PrepareToInvalidateCacheTuple(Relation relation, } } - /* - * Subroutines for warning about reference leaks. These are exported so - * that resowner.c can call them. + * ResourceOwner callbacks */ -void -PrintCatCacheLeakWarning(HeapTuple tuple) +static void +ResOwnerReleaseCatCache(Datum res) +{ + ReleaseCatCache((HeapTuple) DatumGetPointer(res)); +} + +static void +ResOwnerPrintCatCacheLeakWarning(Datum res) { + HeapTuple tuple = (HeapTuple) DatumGetPointer(res); CatCTup *ct = (CatCTup *) (((char *) tuple) - offsetof(CatCTup, tuple)); @@ -2090,9 +2132,17 @@ PrintCatCacheLeakWarning(HeapTuple tuple) ct->refcount); } -void -PrintCatCacheListLeakWarning(CatCList *list) +static void +ResOwnerReleaseCatCacheList(Datum res) { + ReleaseCatCacheList((CatCList *) DatumGetPointer(res)); +} + +static void +ResOwnerPrintCatCacheListLeakWarning(Datum res) +{ + CatCList *list = (CatCList *) DatumGetPointer(res); + elog(WARNING, "cache reference leak: cache %s (%d), list %p has count %d", list->my_cache->cc_relname, list->my_cache->id, list, list->refcount); diff --git a/src/backend/utils/cache/plancache.c b/src/backend/utils/cache/plancache.c index cc04b5b4bef..ecc9296d2d3 100644 --- a/src/backend/utils/cache/plancache.c +++ b/src/backend/utils/cache/plancache.c @@ -69,7 +69,7 @@ #include "tcop/utility.h" #include "utils/inval.h" #include "utils/memutils.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/rls.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -115,6 +115,26 @@ static void PlanCacheRelCallback(Datum arg, Oid relid); static void PlanCacheObjectCallback(Datum arg, int cacheid, uint32 hashvalue); static void PlanCacheSysCallback(Datum arg, int cacheid, uint32 hashvalue); +/* ResourceOwner callbacks to track plancache references */ +static void ResOwnerReleaseCachedPlan(Datum res); +static void ResOwnerPrintPlanCacheLeakWarning(Datum res); + +/* this is exported for ResourceOwnerReleaseAllPlanCacheRefs() */ +ResourceOwnerFuncs planref_resowner_funcs = +{ + .name = "plancache reference", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseCachedPlan, + .PrintLeakWarning = ResOwnerPrintPlanCacheLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberPlanCacheRef(owner, plan) \ + ResourceOwnerRemember(owner, PointerGetDatum(plan), &planref_resowner_funcs) +#define ResourceOwnerForgetPlanCacheRef(owner, plan) \ + ResourceOwnerForget(owner, PointerGetDatum(plan), &planref_resowner_funcs) + + /* GUC parameter */ int plan_cache_mode; @@ -1229,7 +1249,7 @@ GetCachedPlan(CachedPlanSource *plansource, ParamListInfo boundParams, /* Flag the plan as in use by caller */ if (useResOwner) - ResourceOwnerEnlargePlanCacheRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); plan->refcount++; if (useResOwner) ResourceOwnerRememberPlanCacheRef(CurrentResourceOwner, plan); @@ -1392,7 +1412,7 @@ CachedPlanAllowsSimpleValidityCheck(CachedPlanSource *plansource, /* Bump refcount if requested. */ if (owner) { - ResourceOwnerEnlargePlanCacheRefs(owner); + ResourceOwnerEnlarge(owner); plan->refcount++; ResourceOwnerRememberPlanCacheRef(owner, plan); } @@ -1451,7 +1471,7 @@ CachedPlanIsSimplyValid(CachedPlanSource *plansource, CachedPlan *plan, /* It's still good. Bump refcount if requested. */ if (owner) { - ResourceOwnerEnlargePlanCacheRefs(owner); + ResourceOwnerEnlarge(owner); plan->refcount++; ResourceOwnerRememberPlanCacheRef(owner, plan); } @@ -2205,3 +2225,20 @@ ResetPlanCache(void) cexpr->is_valid = false; } } + +/* + * ResourceOwner callbacks + */ + +static void +ResOwnerReleaseCachedPlan(Datum res) +{ + ReleaseCachedPlan((CachedPlan *) DatumGetPointer(res), true); +} + +static void +ResOwnerPrintPlanCacheLeakWarning(Datum res) +{ + elog(WARNING, "plancache reference leak: plan %p not closed", + DatumGetPointer(res)); +} diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 7ef510cd01b..6ca5b50552b 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -78,13 +78,14 @@ #include "storage/smgr.h" #include "utils/array.h" #include "utils/builtins.h" +#include "utils/catcache.h" #include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/relmapper.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -2078,6 +2079,24 @@ RelationIdGetRelation(Oid relationId) * ---------------------------------------------------------------- */ +/* ResourceOwner callbacks to track relcache references */ +static void ResOwnerReleaseRelation(Datum res); +static void ResOwnerPrintRelCacheLeakWarning(Datum res); + +static ResourceOwnerFuncs relref_resowner_funcs = +{ + .name = "relcache reference", + .phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .ReleaseResource = ResOwnerReleaseRelation, + .PrintLeakWarning = ResOwnerPrintRelCacheLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberRelationRef(owner, rel) \ + ResourceOwnerRemember(owner, PointerGetDatum(rel), &relref_resowner_funcs) +#define ResourceOwnerForgetRelationRef(owner, rel) \ + ResourceOwnerForget(owner, PointerGetDatum(rel), &relref_resowner_funcs) + /* * RelationIncrementReferenceCount * Increments relation reference count. @@ -2089,7 +2108,7 @@ RelationIdGetRelation(Oid relationId) void RelationIncrementReferenceCount(Relation rel) { - ResourceOwnerEnlargeRelationRefs(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); rel->rd_refcnt += 1; if (!IsBootstrapProcessingMode()) ResourceOwnerRememberRelationRef(CurrentResourceOwner, rel); @@ -6417,3 +6436,21 @@ unlink_initfile(const char *initfilename, int elevel) initfilename))); } } + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerPrintRelCacheLeakWarning(Datum res) +{ + Relation rel = (Relation) res; + + elog(WARNING, "relcache reference leak: relation \"%s\" not closed", + RelationGetRelationName(rel)); +} + +static void +ResOwnerReleaseRelation(Datum res) +{ + RelationClose((Relation) res); +} diff --git a/src/backend/utils/resowner/README b/src/backend/utils/resowner/README index 2998f6bb362..4ed13afa101 100644 --- a/src/backend/utils/resowner/README +++ b/src/backend/utils/resowner/README @@ -60,13 +60,18 @@ subtransaction or portal. Therefore, the "release" operation on a child ResourceOwner transfers lock ownership to the parent instead of actually releasing the lock, if isCommit is true. -Currently, ResourceOwners contain direct support for recording ownership of -buffer pins, lmgr locks, and catcache, relcache, plancache, tupdesc, and -snapshot references. Other objects can be associated with a ResourceOwner by -recording the address of the owning ResourceOwner in such an object. There is -an API for other modules to get control during ResourceOwner release, so that -they can scan their own data structures to find the objects that need to be -deleted. +ResourceOwner can record ownership of many different kinds of resources. +As of this writing, it's used internally for buffer pins, lmgr locks, and +catcache, relcache, plancache, tupdesc, snapshot, DSM and JIT context +references. ResourceOwner treats all resources the same, and extensions +can define new kinds of resources by filling in a ResourceOwnerFuncs +struct with a suitable callback functions. + +There is also an API for other modules to get control during ResourceOwner +release, so that they can scan their own data structures to find the objects +that need to be deleted. This used to be the only way to register new kinds +of objects with a resource owner; nowadays it easier to write custom +ResourceOwnerFuncs callbacks. Whenever we are inside a transaction, the global variable CurrentResourceOwner shows which resource owner should be assigned diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c index 10f15f6a357..1db1e7cf14b 100644 --- a/src/backend/utils/resowner/resowner.c +++ b/src/backend/utils/resowner/resowner.c @@ -20,75 +20,45 @@ */ #include "postgres.h" -#include "common/cryptohash.h" #include "common/hashfn.h" -#include "jit/jit.h" -#include "storage/bufmgr.h" #include "storage/ipc.h" #include "storage/predicate.h" #include "storage/proc.h" #include "utils/memutils.h" -#include "utils/rel.h" -#include "utils/resowner_private.h" -#include "utils/snapmgr.h" - - -/* - * All resource IDs managed by this code are required to fit into a Datum, - * which is fine since they are generally pointers or integers. - * - * Provide Datum conversion macros for a couple of things that are really - * just "int". - */ -#define FileGetDatum(file) Int32GetDatum(file) -#define DatumGetFile(datum) ((File) DatumGetInt32(datum)) -#define BufferGetDatum(buffer) Int32GetDatum(buffer) -#define DatumGetBuffer(datum) ((Buffer) DatumGetInt32(datum)) +#include "utils/plancache.h" +#include "utils/resowner.h" /* - * ResourceArray is a common structure for storing all types of resource IDs. - * - * We manage small sets of resource IDs by keeping them in a simple array: - * itemsarr[k] holds an ID, for 0 <= k < nitems <= maxitems = capacity. - * - * If a set grows large, we switch over to using open-addressing hashing. - * Then, itemsarr[] is a hash table of "capacity" slots, with each - * slot holding either an ID or "invalidval". nitems is the number of valid - * items present; if it would exceed maxitems, we enlarge the array and - * re-hash. In this mode, maxitems should be rather less than capacity so - * that we don't waste too much time searching for empty slots. + * ResourceElem represents a reference associated with a resource owner. * - * In either mode, lastidx remembers the location of the last item inserted - * or returned by GetAny; this speeds up searches in ResourceArrayRemove. + * All objects managed by this code are required to fit into a Datum, + * which is fine since they are generally pointers or integers. */ -typedef struct ResourceArray +typedef struct ResourceElem { - Datum *itemsarr; /* buffer for storing values */ - Datum invalidval; /* value that is considered invalid */ - uint32 capacity; /* allocated length of itemsarr[] */ - uint32 nitems; /* how many items are stored in items array */ - uint32 maxitems; /* current limit on nitems before enlarging */ - uint32 lastidx; /* index of last item returned by GetAny */ -} ResourceArray; + Datum item; + ResourceOwnerFuncs *kind; +} ResourceElem; /* - * Initially allocated size of a ResourceArray. Must be power of two since - * we'll use (arraysize - 1) as mask for hashing. + * Size of the small fixed-size array to hold most-recently remembered resources. */ -#define RESARRAY_INIT_SIZE 16 +#define RESOWNER_ARRAY_SIZE 8 /* - * When to switch to hashing vs. simple array logic in a ResourceArray. + * Initially allocated size of a ResourceOwner's hash. Must be power of two since + * we'll use (capacity - 1) as mask for hashing. */ -#define RESARRAY_MAX_ARRAY 64 -#define RESARRAY_IS_ARRAY(resarr) ((resarr)->capacity <= RESARRAY_MAX_ARRAY) +#define RESOWNER_HASH_INIT_SIZE 32 /* - * How many items may be stored in a resource array of given capacity. + * How many items may be stored in a hash of given capacity. * When this number is reached, we must resize. */ -#define RESARRAY_MAX_ITEMS(capacity) \ - ((capacity) <= RESARRAY_MAX_ARRAY ? (capacity) : (capacity)/4 * 3) +#define RESOWNER_HASH_MAX_ITEMS(capacity) ((capacity)/4 * 3) + +StaticAssertDecl(RESOWNER_HASH_MAX_ITEMS(RESOWNER_HASH_INIT_SIZE) > RESOWNER_ARRAY_SIZE, + "initial hash size too small compared to array size"); /* * To speed up bulk releasing or reassigning locks from a resource owner to @@ -118,23 +88,33 @@ typedef struct ResourceOwnerData ResourceOwner nextchild; /* next child of same parent */ const char *name; /* name (just for debugging) */ - /* We have built-in support for remembering: */ - ResourceArray bufferarr; /* owned buffers */ - ResourceArray catrefarr; /* catcache references */ - ResourceArray catlistrefarr; /* catcache-list pins */ - ResourceArray relrefarr; /* relcache references */ - ResourceArray planrefarr; /* plancache references */ - ResourceArray tupdescarr; /* tupdesc references */ - ResourceArray snapshotarr; /* snapshot references */ - ResourceArray filearr; /* open temporary files */ - ResourceArray dsmarr; /* dynamic shmem segments */ - ResourceArray jitarr; /* JIT contexts */ - ResourceArray cryptohasharr; /* cryptohash contexts */ + /* + * These structs keep track of the objects registered with this owner. + * + * We manage a small set of references by keeping them in a simple array. + * When the array gets full, all the elements in the array are moved to a + * hash table. This way, the array always contains a few most recently + * remembered references. To find a particular reference, you need to + * search both the array and the hash table. + */ + ResourceElem arr[RESOWNER_ARRAY_SIZE]; + uint32 narr; /* how many items are stored in the array */ + + /* + * The hash table. Uses open-addressing. 'nhash' is the number of items + * present; if it would exceed 'grow_at', we enlarge it and re-hash. + * 'grow_at' should be rather less than 'capacity' so that we don't waste + * too much time searching for empty slots. + */ + ResourceElem *hash; + uint32 nhash; /* how many items are stored in the hash */ + uint32 capacity; /* allocated length of hash[] */ + uint32 grow_at; /* grow hash when reach this */ /* We can remember up to MAX_RESOWNER_LOCKS references to local locks. */ int nlocks; /* number of owned locks */ LOCALLOCK *locks[MAX_RESOWNER_LOCKS]; /* list of owned locks */ -} ResourceOwnerData; +} ResourceOwnerData; /***************************************************************************** @@ -146,6 +126,18 @@ ResourceOwner CurTransactionResourceOwner = NULL; ResourceOwner TopTransactionResourceOwner = NULL; ResourceOwner AuxProcessResourceOwner = NULL; +/* #define RESOWNER_STATS */ +/* #define RESOWNER_TRACE */ + +#ifdef RESOWNER_STATS +static int narray_lookups = 0; +static int nhash_lookups = 0; +#endif + +#ifdef RESOWNER_TRACE +static int resowner_trace_counter = 0; +#endif + /* * List of add-on callbacks for resource releasing */ @@ -160,296 +152,315 @@ static ResourceReleaseCallbackItem *ResourceRelease_callbacks = NULL; /* Internal routines */ -static void ResourceArrayInit(ResourceArray *resarr, Datum invalidval); -static void ResourceArrayEnlarge(ResourceArray *resarr); -static void ResourceArrayAdd(ResourceArray *resarr, Datum value); -static bool ResourceArrayRemove(ResourceArray *resarr, Datum value); -static bool ResourceArrayGetAny(ResourceArray *resarr, Datum *value); -static void ResourceArrayFree(ResourceArray *resarr); static void ResourceOwnerReleaseInternal(ResourceOwner owner, ResourceReleasePhase phase, bool isCommit, bool isTopLevel); static void ReleaseAuxProcessResourcesCallback(int code, Datum arg); -static void PrintRelCacheLeakWarning(Relation rel); -static void PrintPlanCacheLeakWarning(CachedPlan *plan); -static void PrintTupleDescLeakWarning(TupleDesc tupdesc); -static void PrintSnapshotLeakWarning(Snapshot snapshot); -static void PrintFileLeakWarning(File file); -static void PrintDSMLeakWarning(dsm_segment *seg); -static void PrintCryptoHashLeakWarning(Datum handle); /***************************************************************************** * INTERNAL ROUTINES * *****************************************************************************/ +static inline uint32 +hash_resource_elem(Datum value, ResourceOwnerFuncs *kind) +{ + Datum data[2]; + + data[0] = value; + data[1] = PointerGetDatum(kind); + + return hash_bytes((unsigned char *) &data, 2 * SIZEOF_DATUM); +} -/* - * Initialize a ResourceArray - */ static void -ResourceArrayInit(ResourceArray *resarr, Datum invalidval) +ResourceOwnerAddToHash(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind) { - /* Assert it's empty */ - Assert(resarr->itemsarr == NULL); - Assert(resarr->capacity == 0); - Assert(resarr->nitems == 0); - Assert(resarr->maxitems == 0); - /* Remember the appropriate "invalid" value */ - resarr->invalidval = invalidval; - /* We don't allocate any storage until needed */ + /* Insert into first free slot at or after hash location. */ + uint32 mask = owner->capacity - 1; + uint32 idx; + + idx = hash_resource_elem(value, kind) & mask; + for (;;) + { + if (owner->hash[idx].kind == NULL) + break; + idx = (idx + 1) & mask; + } + owner->hash[idx].item = value; + owner->hash[idx].kind = kind; + owner->nhash++; } /* - * Make sure there is room for at least one more resource in an array. - * - * This is separate from actually inserting a resource because if we run out - * of memory, it's critical to do so *before* acquiring the resource. + * Call the ReleaseResource callback on entries with given 'phase'. */ static void -ResourceArrayEnlarge(ResourceArray *resarr) +ResourceOwnerReleaseAll(ResourceOwner owner, ResourceReleasePhase phase, + bool printLeakWarnings) { - uint32 i, - oldcap, - newcap; - Datum *olditemsarr; - Datum *newitemsarr; - - if (resarr->nitems < resarr->maxitems) - return; /* no work needed */ + bool found; + int capacity; - olditemsarr = resarr->itemsarr; - oldcap = resarr->capacity; - - /* Double the capacity of the array (capacity must stay a power of 2!) */ - newcap = (oldcap > 0) ? oldcap * 2 : RESARRAY_INIT_SIZE; - newitemsarr = (Datum *) MemoryContextAlloc(TopMemoryContext, - newcap * sizeof(Datum)); - for (i = 0; i < newcap; i++) - newitemsarr[i] = resarr->invalidval; + /* First handle all the entries in the array. */ + do + { + found = false; + for (int i = 0; i < owner->narr; i++) + { + if (owner->arr[i].kind->phase == phase) + { + Datum value = owner->arr[i].item; + ResourceOwnerFuncs *kind = owner->arr[i].kind; - /* We assume we can't fail below this point, so OK to scribble on resarr */ - resarr->itemsarr = newitemsarr; - resarr->capacity = newcap; - resarr->maxitems = RESARRAY_MAX_ITEMS(newcap); - resarr->nitems = 0; + if (printLeakWarnings) + kind->PrintLeakWarning(value); + kind->ReleaseResource(value); + found = true; + } + } - if (olditemsarr != NULL) - { /* - * Transfer any pre-existing entries into the new array; they don't - * necessarily go where they were before, so this simple logic is the - * best way. Note that if we were managing the set as a simple array, - * the entries after nitems are garbage, but that shouldn't matter - * because we won't get here unless nitems was equal to oldcap. + * If any resources were released, check again because some of the + * elements might have been moved by the callbacks. We don't want to + * miss them. */ - for (i = 0; i < oldcap; i++) + } while (found && owner->narr > 0); + + /* Ok, the array has now been handled. Then the hash */ + do + { + capacity = owner->capacity; + for (int idx = 0; idx < capacity; idx++) { - if (olditemsarr[i] != resarr->invalidval) - ResourceArrayAdd(resarr, olditemsarr[i]); + while (owner->hash[idx].kind != NULL && + owner->hash[idx].kind->phase == phase) + { + Datum value = owner->hash[idx].item; + ResourceOwnerFuncs *kind = owner->hash[idx].kind; + + if (printLeakWarnings) + kind->PrintLeakWarning(value); + kind->ReleaseResource(value); + + /* + * If the resource is remembered more than once in this + * resource owner, the ReleaseResource callback might've + * released a different copy of it. Because of that, loop to + * check the same index again. + */ + } } - /* And release old array. */ - pfree(olditemsarr); - } - - Assert(resarr->nitems < resarr->maxitems); + /* + * It's possible that the callbacks acquired more resources, causing + * the hash table to grow and the existing entries to be moved around. + * If that happened, scan the hash table again, so that we don't miss + * entries that were moved. (XXX: I'm not sure if any of the callbacks + * actually do that, but this is cheap to check, and better safe than + * sorry.) + */ + Assert(owner->capacity >= capacity); + } while (capacity != owner->capacity); } + +/***************************************************************************** + * EXPORTED ROUTINES * + *****************************************************************************/ + + /* - * Add a resource to ResourceArray + * ResourceOwnerCreate + * Create an empty ResourceOwner. * - * Caller must have previously done ResourceArrayEnlarge() + * All ResourceOwner objects are kept in TopMemoryContext, since they should + * only be freed explicitly. */ -static void -ResourceArrayAdd(ResourceArray *resarr, Datum value) +ResourceOwner +ResourceOwnerCreate(ResourceOwner parent, const char *name) { - uint32 idx; + ResourceOwner owner; - Assert(value != resarr->invalidval); - Assert(resarr->nitems < resarr->maxitems); + owner = (ResourceOwner) MemoryContextAllocZero(TopMemoryContext, + sizeof(ResourceOwnerData)); + owner->name = name; - if (RESARRAY_IS_ARRAY(resarr)) + if (parent) { - /* Append to linear array. */ - idx = resarr->nitems; + owner->parent = parent; + owner->nextchild = parent->firstchild; + parent->firstchild = owner; } - else - { - /* Insert into first free slot at or after hash location. */ - uint32 mask = resarr->capacity - 1; - idx = DatumGetUInt32(hash_any((void *) &value, sizeof(value))) & mask; - for (;;) - { - if (resarr->itemsarr[idx] == resarr->invalidval) - break; - idx = (idx + 1) & mask; - } - } - resarr->lastidx = idx; - resarr->itemsarr[idx] = value; - resarr->nitems++; +#ifdef RESOWNER_TRACE + elog(LOG, "CREATE %d: %p %s", + resowner_trace_counter++, owner, name); +#endif + + return owner; } /* - * Remove a resource from ResourceArray - * - * Returns true on success, false if resource was not found. + * Make sure there is room for at least one more resource in an array. * - * Note: if same resource ID appears more than once, one instance is removed. + * This is separate from actually inserting a resource because if we run out + * of memory, it's critical to do so *before* acquiring the resource. */ -static bool -ResourceArrayRemove(ResourceArray *resarr, Datum value) +void +ResourceOwnerEnlarge(ResourceOwner owner) { - uint32 i, - idx, - lastidx = resarr->lastidx; - - Assert(value != resarr->invalidval); + if (owner->narr < RESOWNER_ARRAY_SIZE) + return; /* no work needed */ - /* Search through all items, but try lastidx first. */ - if (RESARRAY_IS_ARRAY(resarr)) + /* Is there space in the hash? If not, enlarge it. */ + if (owner->narr + owner->nhash >= owner->grow_at) { - if (lastidx < resarr->nitems && - resarr->itemsarr[lastidx] == value) - { - resarr->itemsarr[lastidx] = resarr->itemsarr[resarr->nitems - 1]; - resarr->nitems--; - /* Update lastidx to make reverse-order removals fast. */ - resarr->lastidx = resarr->nitems - 1; - return true; - } - for (i = 0; i < resarr->nitems; i++) + uint32 i, + oldcap, + newcap; + ResourceElem *oldhash; + ResourceElem *newhash; + + oldhash = owner->hash; + oldcap = owner->capacity; + + /* Double the capacity (it must stay a power of 2!) */ + newcap = (oldcap > 0) ? oldcap * 2 : RESOWNER_HASH_INIT_SIZE; + newhash = (ResourceElem *) MemoryContextAllocZero(TopMemoryContext, + newcap * sizeof(ResourceElem)); + + /* + * We assume we can't fail below this point, so OK to scribble on the + * owner + */ + owner->hash = newhash; + owner->capacity = newcap; + owner->grow_at = RESOWNER_HASH_MAX_ITEMS(newcap); + owner->nhash = 0; + + if (oldhash != NULL) { - if (resarr->itemsarr[i] == value) + /* + * Transfer any pre-existing entries into the new hash table; they + * don't necessarily go where they were before, so this simple + * logic is the best way. + */ + for (i = 0; i < oldcap; i++) { - resarr->itemsarr[i] = resarr->itemsarr[resarr->nitems - 1]; - resarr->nitems--; - /* Update lastidx to make reverse-order removals fast. */ - resarr->lastidx = resarr->nitems - 1; - return true; + if (oldhash[i].kind != NULL) + ResourceOwnerAddToHash(owner, oldhash[i].item, oldhash[i].kind); } + + /* And release old hash table. */ + pfree(oldhash); } } - else - { - uint32 mask = resarr->capacity - 1; - if (lastidx < resarr->capacity && - resarr->itemsarr[lastidx] == value) - { - resarr->itemsarr[lastidx] = resarr->invalidval; - resarr->nitems--; - return true; - } - idx = DatumGetUInt32(hash_any((void *) &value, sizeof(value))) & mask; - for (i = 0; i < resarr->capacity; i++) - { - if (resarr->itemsarr[idx] == value) - { - resarr->itemsarr[idx] = resarr->invalidval; - resarr->nitems--; - return true; - } - idx = (idx + 1) & mask; - } + /* Move items from the array to the hash */ + Assert(owner->narr == RESOWNER_ARRAY_SIZE); + for (int i = 0; i < owner->narr; i++) + { + ResourceOwnerAddToHash(owner, owner->arr[i].item, owner->arr[i].kind); } + owner->narr = 0; - return false; + Assert(owner->nhash < owner->grow_at); } /* - * Get any convenient entry in a ResourceArray. + * Remember that an object is owner by a ReourceOwner * - * "Convenient" is defined as "easy for ResourceArrayRemove to remove"; - * we help that along by setting lastidx to match. This avoids O(N^2) cost - * when removing all ResourceArray items during ResourceOwner destruction. - * - * Returns true if we found an element, or false if the array is empty. + * Caller must have previously done ResourceOwnerEnlarge() */ -static bool -ResourceArrayGetAny(ResourceArray *resarr, Datum *value) +void +ResourceOwnerRemember(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind) { - if (resarr->nitems == 0) - return false; + uint32 idx; - if (RESARRAY_IS_ARRAY(resarr)) - { - /* Linear array: just return the first element. */ - resarr->lastidx = 0; - } - else - { - /* Hash: search forward from wherever we were last. */ - uint32 mask = resarr->capacity - 1; +#ifdef RESOWNER_TRACE + elog(LOG, "REMEMBER %d: owner %p value " UINT64_FORMAT ", kind: %s", + resowner_trace_counter++, owner, DatumGetUInt64(value), kind->name); +#endif - for (;;) - { - resarr->lastidx &= mask; - if (resarr->itemsarr[resarr->lastidx] != resarr->invalidval) - break; - resarr->lastidx++; - } - } + Assert(owner->narr < RESOWNER_ARRAY_SIZE); - *value = resarr->itemsarr[resarr->lastidx]; - return true; + /* Append to linear array. */ + idx = owner->narr; + owner->arr[idx].item = value; + owner->arr[idx].kind = kind; + owner->narr++; } /* - * Trash a ResourceArray (we don't care about its state after this) + * Forget that an object is owned by a ResourceOwner + * + * Returns true on success. If the resource was not found, returns false, + * and calls kind->ForgetError callback. + * + * Note: if same resource ID is associated with the ResourceOwner more than once, + * one instance is removed. */ -static void -ResourceArrayFree(ResourceArray *resarr) +void +ResourceOwnerForget(ResourceOwner owner, Datum value, ResourceOwnerFuncs *kind) { - if (resarr->itemsarr) - pfree(resarr->itemsarr); -} - + uint32 i, + idx; -/***************************************************************************** - * EXPORTED ROUTINES * - *****************************************************************************/ +#ifdef RESOWNER_TRACE + elog(LOG, "FORGET %d: owner %p value " UINT64_FORMAT ", kind: %s", + resowner_trace_counter++, owner, DatumGetUInt64(value), kind->name); +#endif + /* Search through all items, but check the array first. */ + for (i = 0; i < owner->narr; i++) + { + if (owner->arr[i].item == value && + owner->arr[i].kind == kind) + { + owner->arr[i] = owner->arr[owner->narr - 1]; + owner->narr--; -/* - * ResourceOwnerCreate - * Create an empty ResourceOwner. - * - * All ResourceOwner objects are kept in TopMemoryContext, since they should - * only be freed explicitly. - */ -ResourceOwner -ResourceOwnerCreate(ResourceOwner parent, const char *name) -{ - ResourceOwner owner; +#ifdef RESOWNER_STATS + narray_lookups++; +#endif - owner = (ResourceOwner) MemoryContextAllocZero(TopMemoryContext, - sizeof(ResourceOwnerData)); - owner->name = name; + return; + } + } - if (parent) + /* Search hash */ + if (owner->nhash > 0) { - owner->parent = parent; - owner->nextchild = parent->firstchild; - parent->firstchild = owner; - } + uint32 mask = owner->capacity - 1; - ResourceArrayInit(&(owner->bufferarr), BufferGetDatum(InvalidBuffer)); - ResourceArrayInit(&(owner->catrefarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->catlistrefarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->relrefarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->planrefarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->tupdescarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->snapshotarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->filearr), FileGetDatum(-1)); - ResourceArrayInit(&(owner->dsmarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->jitarr), PointerGetDatum(NULL)); - ResourceArrayInit(&(owner->cryptohasharr), PointerGetDatum(NULL)); + idx = hash_resource_elem(value, kind) & mask; + for (i = 0; i < owner->capacity; i++) + { + if (owner->hash[idx].item == value && + owner->hash[idx].kind == kind) + { + owner->hash[idx].item = (Datum) 0; + owner->hash[idx].kind = NULL; + owner->nhash--; + +#ifdef RESOWNER_STATS + nhash_lookups++; +#endif + return; + } + idx = (idx + 1) & mask; + } + } - return owner; + /* + * Use %p to print the reference, since most objects tracked by a resource + * owner are pointers. It's a bit misleading if it's not a pointer, but + * this is a programmer error, anyway. + */ + elog(ERROR, "%s %p is not owned by resource owner %s", + kind->name, DatumGetPointer(value), owner->name); } /* @@ -486,6 +497,15 @@ ResourceOwnerRelease(ResourceOwner owner, { /* There's not currently any setup needed before recursing */ ResourceOwnerReleaseInternal(owner, phase, isCommit, isTopLevel); + +#ifdef RESOWNER_STATS + if (isTopLevel) + { + elog(LOG, "RESOWNER STATS: lookups: array %d, hash %d", narray_lookups, nhash_lookups); + narray_lookups = 0; + nhash_lookups = 0; + } +#endif } static void @@ -497,7 +517,6 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, ResourceOwner child; ResourceOwner save; ResourceReleaseCallbackItem *item; - Datum foundres; /* Recurse to handle descendants */ for (child = owner->firstchild; child != NULL; child = child->nextchild) @@ -513,61 +532,13 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, if (phase == RESOURCE_RELEASE_BEFORE_LOCKS) { /* - * Release buffer pins. Note that ReleaseBuffer will remove the - * buffer entry from our array, so we just have to iterate till there - * are none. + * Release all references that need to be released before the locks. * - * During a commit, there shouldn't be any remaining pins --- that - * would indicate failure to clean up the executor correctly --- so - * issue warnings. In the abort case, just clean up quietly. + * During a commit, there shouldn't be any remaining references --- + * that would indicate failure to clean up the executor correctly --- + * so issue warnings. In the abort case, just clean up quietly. */ - while (ResourceArrayGetAny(&(owner->bufferarr), &foundres)) - { - Buffer res = DatumGetBuffer(foundres); - - if (isCommit) - PrintBufferLeakWarning(res); - ReleaseBuffer(res); - } - - /* Ditto for relcache references */ - while (ResourceArrayGetAny(&(owner->relrefarr), &foundres)) - { - Relation res = (Relation) DatumGetPointer(foundres); - - if (isCommit) - PrintRelCacheLeakWarning(res); - RelationClose(res); - } - - /* Ditto for dynamic shared memory segments */ - while (ResourceArrayGetAny(&(owner->dsmarr), &foundres)) - { - dsm_segment *res = (dsm_segment *) DatumGetPointer(foundres); - - if (isCommit) - PrintDSMLeakWarning(res); - dsm_detach(res); - } - - /* Ditto for JIT contexts */ - while (ResourceArrayGetAny(&(owner->jitarr), &foundres)) - { - JitContext *context = (JitContext *) PointerGetDatum(foundres); - - jit_release_context(context); - } - - /* Ditto for cryptohash contexts */ - while (ResourceArrayGetAny(&(owner->cryptohasharr), &foundres)) - { - pg_cryptohash_ctx *context = - (pg_cryptohash_ctx *) PointerGetDatum(foundres); - - if (isCommit) - PrintCryptoHashLeakWarning(foundres); - pg_cryptohash_free(context); - } + ResourceOwnerReleaseAll(owner, phase, isCommit); } else if (phase == RESOURCE_RELEASE_LOCKS) { @@ -576,7 +547,7 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, /* * For a top-level xact we are going to release all locks (or at * least all non-session locks), so just do a single lmgr call at - * the top of the recursion. + * the top of the recursion */ if (owner == TopTransactionResourceOwner) { @@ -620,70 +591,9 @@ ResourceOwnerReleaseInternal(ResourceOwner owner, else if (phase == RESOURCE_RELEASE_AFTER_LOCKS) { /* - * Release catcache references. Note that ReleaseCatCache will remove - * the catref entry from our array, so we just have to iterate till - * there are none. - * - * As with buffer pins, warn if any are left at commit time. + * Release all references that need to be released after the locks. */ - while (ResourceArrayGetAny(&(owner->catrefarr), &foundres)) - { - HeapTuple res = (HeapTuple) DatumGetPointer(foundres); - - if (isCommit) - PrintCatCacheLeakWarning(res); - ReleaseCatCache(res); - } - - /* Ditto for catcache lists */ - while (ResourceArrayGetAny(&(owner->catlistrefarr), &foundres)) - { - CatCList *res = (CatCList *) DatumGetPointer(foundres); - - if (isCommit) - PrintCatCacheListLeakWarning(res); - ReleaseCatCacheList(res); - } - - /* Ditto for plancache references */ - while (ResourceArrayGetAny(&(owner->planrefarr), &foundres)) - { - CachedPlan *res = (CachedPlan *) DatumGetPointer(foundres); - - if (isCommit) - PrintPlanCacheLeakWarning(res); - ReleaseCachedPlan(res, true); - } - - /* Ditto for tupdesc references */ - while (ResourceArrayGetAny(&(owner->tupdescarr), &foundres)) - { - TupleDesc res = (TupleDesc) DatumGetPointer(foundres); - - if (isCommit) - PrintTupleDescLeakWarning(res); - DecrTupleDescRefCount(res); - } - - /* Ditto for snapshot references */ - while (ResourceArrayGetAny(&(owner->snapshotarr), &foundres)) - { - Snapshot res = (Snapshot) DatumGetPointer(foundres); - - if (isCommit) - PrintSnapshotLeakWarning(res); - UnregisterSnapshot(res); - } - - /* Ditto for temporary files */ - while (ResourceArrayGetAny(&(owner->filearr), &foundres)) - { - File res = DatumGetFile(foundres); - - if (isCommit) - PrintFileLeakWarning(res); - FileClose(res); - } + ResourceOwnerReleaseAll(owner, phase, isCommit); } /* Let add-on modules get a chance too */ @@ -704,16 +614,48 @@ void ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner) { ResourceOwner save; - Datum foundres; save = CurrentResourceOwner; CurrentResourceOwner = owner; - while (ResourceArrayGetAny(&(owner->planrefarr), &foundres)) + + /* array first */ + for (int i = 0; i < owner->narr; i++) + { + if (owner->arr[i].kind == &planref_resowner_funcs) + { + CachedPlan *planref = (CachedPlan *) DatumGetPointer(owner->arr[i].item); + + owner->arr[i] = owner->arr[owner->narr - 1]; + owner->narr--; + i--; + + /* + * pass 'false' because we already removed the entry from the + * resowner + */ + ReleaseCachedPlan(planref, false); + } + } + + /* Then hash */ + for (int i = 0; i < owner->capacity; i++) { - CachedPlan *res = (CachedPlan *) DatumGetPointer(foundres); + if (owner->hash[i].kind == &planref_resowner_funcs) + { + CachedPlan *planref = (CachedPlan *) DatumGetPointer(owner->hash[i].item); - ReleaseCachedPlan(res, true); + owner->hash[i].item = (Datum) 0; + owner->hash[i].kind = NULL; + owner->nhash--; + + /* + * pass 'false' because we already removed the entry from the + * resowner + */ + ReleaseCachedPlan(planref, false); + } } + CurrentResourceOwner = save; } @@ -730,19 +672,15 @@ ResourceOwnerDelete(ResourceOwner owner) Assert(owner != CurrentResourceOwner); /* And it better not own any resources, either */ - Assert(owner->bufferarr.nitems == 0); - Assert(owner->catrefarr.nitems == 0); - Assert(owner->catlistrefarr.nitems == 0); - Assert(owner->relrefarr.nitems == 0); - Assert(owner->planrefarr.nitems == 0); - Assert(owner->tupdescarr.nitems == 0); - Assert(owner->snapshotarr.nitems == 0); - Assert(owner->filearr.nitems == 0); - Assert(owner->dsmarr.nitems == 0); - Assert(owner->jitarr.nitems == 0); - Assert(owner->cryptohasharr.nitems == 0); + Assert(owner->narr == 0); + Assert(owner->nhash == 0); Assert(owner->nlocks == 0 || owner->nlocks == MAX_RESOWNER_LOCKS + 1); +#ifdef RESOWNER_TRACE + elog(LOG, "DELETE %d: %p %s", + resowner_trace_counter++, owner, owner->name); +#endif + /* * Delete children. The recursive call will delink the child from me, so * just iterate as long as there is a child. @@ -758,18 +696,8 @@ ResourceOwnerDelete(ResourceOwner owner) ResourceOwnerNewParent(owner, NULL); /* And free the object. */ - ResourceArrayFree(&(owner->bufferarr)); - ResourceArrayFree(&(owner->catrefarr)); - ResourceArrayFree(&(owner->catlistrefarr)); - ResourceArrayFree(&(owner->relrefarr)); - ResourceArrayFree(&(owner->planrefarr)); - ResourceArrayFree(&(owner->tupdescarr)); - ResourceArrayFree(&(owner->snapshotarr)); - ResourceArrayFree(&(owner->filearr)); - ResourceArrayFree(&(owner->dsmarr)); - ResourceArrayFree(&(owner->jitarr)); - ResourceArrayFree(&(owner->cryptohasharr)); - + if (owner->hash) + pfree(owner->hash); pfree(owner); } @@ -827,11 +755,10 @@ ResourceOwnerNewParent(ResourceOwner owner, /* * Register or deregister callback functions for resource cleanup * - * These functions are intended for use by dynamically loaded modules. - * For built-in modules we generally just hardwire the appropriate calls. - * - * Note that the callback occurs post-commit or post-abort, so the callback - * functions can only do noncritical cleanup. + * These functions can be used by dynamically loaded modules. These used + * to be the only way for an extension to register custom resource types + * with a resource owner, but nowadays it is easier to define a new + * ResourceOwnerFuncs instance with custom callbacks. */ void RegisterResourceReleaseCallback(ResourceReleaseCallback callback, void *arg) @@ -922,44 +849,6 @@ ReleaseAuxProcessResourcesCallback(int code, Datum arg) ReleaseAuxProcessResources(isCommit); } - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * buffer array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeBuffers(ResourceOwner owner) -{ - /* We used to allow pinning buffers without a resowner, but no more */ - Assert(owner != NULL); - ResourceArrayEnlarge(&(owner->bufferarr)); -} - -/* - * Remember that a buffer pin is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeBuffers() - */ -void -ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer) -{ - ResourceArrayAdd(&(owner->bufferarr), BufferGetDatum(buffer)); -} - -/* - * Forget that a buffer pin is owned by a ResourceOwner - */ -void -ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer) -{ - if (!ResourceArrayRemove(&(owner->bufferarr), BufferGetDatum(buffer))) - elog(ERROR, "buffer %d is not owned by resource owner %s", - buffer, owner->name); -} - /* * Remember that a Local Lock is owned by a ResourceOwner * @@ -1011,424 +900,3 @@ ResourceOwnerForgetLock(ResourceOwner owner, LOCALLOCK *locallock) elog(ERROR, "lock reference %p is not owned by resource owner %s", locallock, owner->name); } - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * catcache reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeCatCacheRefs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->catrefarr)); -} - -/* - * Remember that a catcache reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeCatCacheRefs() - */ -void -ResourceOwnerRememberCatCacheRef(ResourceOwner owner, HeapTuple tuple) -{ - ResourceArrayAdd(&(owner->catrefarr), PointerGetDatum(tuple)); -} - -/* - * Forget that a catcache reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetCatCacheRef(ResourceOwner owner, HeapTuple tuple) -{ - if (!ResourceArrayRemove(&(owner->catrefarr), PointerGetDatum(tuple))) - elog(ERROR, "catcache reference %p is not owned by resource owner %s", - tuple, owner->name); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * catcache-list reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeCatCacheListRefs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->catlistrefarr)); -} - -/* - * Remember that a catcache-list reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeCatCacheListRefs() - */ -void -ResourceOwnerRememberCatCacheListRef(ResourceOwner owner, CatCList *list) -{ - ResourceArrayAdd(&(owner->catlistrefarr), PointerGetDatum(list)); -} - -/* - * Forget that a catcache-list reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetCatCacheListRef(ResourceOwner owner, CatCList *list) -{ - if (!ResourceArrayRemove(&(owner->catlistrefarr), PointerGetDatum(list))) - elog(ERROR, "catcache list reference %p is not owned by resource owner %s", - list, owner->name); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * relcache reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeRelationRefs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->relrefarr)); -} - -/* - * Remember that a relcache reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeRelationRefs() - */ -void -ResourceOwnerRememberRelationRef(ResourceOwner owner, Relation rel) -{ - ResourceArrayAdd(&(owner->relrefarr), PointerGetDatum(rel)); -} - -/* - * Forget that a relcache reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetRelationRef(ResourceOwner owner, Relation rel) -{ - if (!ResourceArrayRemove(&(owner->relrefarr), PointerGetDatum(rel))) - elog(ERROR, "relcache reference %s is not owned by resource owner %s", - RelationGetRelationName(rel), owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintRelCacheLeakWarning(Relation rel) -{ - elog(WARNING, "relcache reference leak: relation \"%s\" not closed", - RelationGetRelationName(rel)); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * plancache reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargePlanCacheRefs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->planrefarr)); -} - -/* - * Remember that a plancache reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargePlanCacheRefs() - */ -void -ResourceOwnerRememberPlanCacheRef(ResourceOwner owner, CachedPlan *plan) -{ - ResourceArrayAdd(&(owner->planrefarr), PointerGetDatum(plan)); -} - -/* - * Forget that a plancache reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetPlanCacheRef(ResourceOwner owner, CachedPlan *plan) -{ - if (!ResourceArrayRemove(&(owner->planrefarr), PointerGetDatum(plan))) - elog(ERROR, "plancache reference %p is not owned by resource owner %s", - plan, owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintPlanCacheLeakWarning(CachedPlan *plan) -{ - elog(WARNING, "plancache reference leak: plan %p not closed", plan); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * tupdesc reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeTupleDescs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->tupdescarr)); -} - -/* - * Remember that a tupdesc reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeTupleDescs() - */ -void -ResourceOwnerRememberTupleDesc(ResourceOwner owner, TupleDesc tupdesc) -{ - ResourceArrayAdd(&(owner->tupdescarr), PointerGetDatum(tupdesc)); -} - -/* - * Forget that a tupdesc reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetTupleDesc(ResourceOwner owner, TupleDesc tupdesc) -{ - if (!ResourceArrayRemove(&(owner->tupdescarr), PointerGetDatum(tupdesc))) - elog(ERROR, "tupdesc reference %p is not owned by resource owner %s", - tupdesc, owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintTupleDescLeakWarning(TupleDesc tupdesc) -{ - elog(WARNING, - "TupleDesc reference leak: TupleDesc %p (%u,%d) still referenced", - tupdesc, tupdesc->tdtypeid, tupdesc->tdtypmod); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * snapshot reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeSnapshots(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->snapshotarr)); -} - -/* - * Remember that a snapshot reference is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeSnapshots() - */ -void -ResourceOwnerRememberSnapshot(ResourceOwner owner, Snapshot snapshot) -{ - ResourceArrayAdd(&(owner->snapshotarr), PointerGetDatum(snapshot)); -} - -/* - * Forget that a snapshot reference is owned by a ResourceOwner - */ -void -ResourceOwnerForgetSnapshot(ResourceOwner owner, Snapshot snapshot) -{ - if (!ResourceArrayRemove(&(owner->snapshotarr), PointerGetDatum(snapshot))) - elog(ERROR, "snapshot reference %p is not owned by resource owner %s", - snapshot, owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintSnapshotLeakWarning(Snapshot snapshot) -{ - elog(WARNING, "Snapshot reference leak: Snapshot %p still referenced", - snapshot); -} - - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * files reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeFiles(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->filearr)); -} - -/* - * Remember that a temporary file is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeFiles() - */ -void -ResourceOwnerRememberFile(ResourceOwner owner, File file) -{ - ResourceArrayAdd(&(owner->filearr), FileGetDatum(file)); -} - -/* - * Forget that a temporary file is owned by a ResourceOwner - */ -void -ResourceOwnerForgetFile(ResourceOwner owner, File file) -{ - if (!ResourceArrayRemove(&(owner->filearr), FileGetDatum(file))) - elog(ERROR, "temporary file %d is not owned by resource owner %s", - file, owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintFileLeakWarning(File file) -{ - elog(WARNING, "temporary file leak: File %d still referenced", - file); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * dynamic shmem segment reference array. - * - * This is separate from actually inserting an entry because if we run out - * of memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeDSMs(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->dsmarr)); -} - -/* - * Remember that a dynamic shmem segment is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeDSMs() - */ -void -ResourceOwnerRememberDSM(ResourceOwner owner, dsm_segment *seg) -{ - ResourceArrayAdd(&(owner->dsmarr), PointerGetDatum(seg)); -} - -/* - * Forget that a dynamic shmem segment is owned by a ResourceOwner - */ -void -ResourceOwnerForgetDSM(ResourceOwner owner, dsm_segment *seg) -{ - if (!ResourceArrayRemove(&(owner->dsmarr), PointerGetDatum(seg))) - elog(ERROR, "dynamic shared memory segment %u is not owned by resource owner %s", - dsm_segment_handle(seg), owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintDSMLeakWarning(dsm_segment *seg) -{ - elog(WARNING, "dynamic shared memory leak: segment %u still referenced", - dsm_segment_handle(seg)); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * JIT context reference array. - * - * This is separate from actually inserting an entry because if we run out of - * memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeJIT(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->jitarr)); -} - -/* - * Remember that a JIT context is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeJIT() - */ -void -ResourceOwnerRememberJIT(ResourceOwner owner, Datum handle) -{ - ResourceArrayAdd(&(owner->jitarr), handle); -} - -/* - * Forget that a JIT context is owned by a ResourceOwner - */ -void -ResourceOwnerForgetJIT(ResourceOwner owner, Datum handle) -{ - if (!ResourceArrayRemove(&(owner->jitarr), handle)) - elog(ERROR, "JIT context %p is not owned by resource owner %s", - DatumGetPointer(handle), owner->name); -} - -/* - * Make sure there is room for at least one more entry in a ResourceOwner's - * cryptohash context reference array. - * - * This is separate from actually inserting an entry because if we run out of - * memory, it's critical to do so *before* acquiring the resource. - */ -void -ResourceOwnerEnlargeCryptoHash(ResourceOwner owner) -{ - ResourceArrayEnlarge(&(owner->cryptohasharr)); -} - -/* - * Remember that a cryptohash context is owned by a ResourceOwner - * - * Caller must have previously done ResourceOwnerEnlargeCryptoHash() - */ -void -ResourceOwnerRememberCryptoHash(ResourceOwner owner, Datum handle) -{ - ResourceArrayAdd(&(owner->cryptohasharr), handle); -} - -/* - * Forget that a cryptohash context is owned by a ResourceOwner - */ -void -ResourceOwnerForgetCryptoHash(ResourceOwner owner, Datum handle) -{ - if (!ResourceArrayRemove(&(owner->cryptohasharr), handle)) - elog(ERROR, "cryptohash context %p is not owned by resource owner %s", - DatumGetPointer(handle), owner->name); -} - -/* - * Debugging subroutine - */ -static void -PrintCryptoHashLeakWarning(Datum handle) -{ - elog(WARNING, "cryptohash context reference leak: context %p still referenced", - DatumGetPointer(handle)); -} diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index ae16c3ed7d6..3ded16f6fc8 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -66,7 +66,7 @@ #include "utils/memutils.h" #include "utils/old_snapshot.h" #include "utils/rel.h" -#include "utils/resowner_private.h" +#include "utils/resowner.h" #include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/timestamp.h" @@ -174,6 +174,24 @@ static Snapshot CopySnapshot(Snapshot snapshot); static void FreeSnapshot(Snapshot snapshot); static void SnapshotResetXmin(void); +/* ResourceOwner callbacks to track snapshot references */ +static void ResOwnerReleaseSnapshot(Datum res); +static void ResOwnerPrintSnapshotLeakWarning(Datum res); + +static ResourceOwnerFuncs snapshot_resowner_funcs = +{ + .name = "snapshot reference", + .phase = RESOURCE_RELEASE_AFTER_LOCKS, + .ReleaseResource = ResOwnerReleaseSnapshot, + .PrintLeakWarning = ResOwnerPrintSnapshotLeakWarning +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberSnapshot(owner, snap) \ + ResourceOwnerRemember(owner, PointerGetDatum(snap), &snapshot_resowner_funcs) +#define ResourceOwnerForgetSnapshot(owner, snap) \ + ResourceOwnerForget(owner, PointerGetDatum(snap), &snapshot_resowner_funcs) + /* * Snapshot fields to be serialized. * @@ -831,7 +849,7 @@ RegisterSnapshotOnOwner(Snapshot snapshot, ResourceOwner owner) snap = snapshot->copied ? snapshot : CopySnapshot(snapshot); /* and tell resowner.c about it */ - ResourceOwnerEnlargeSnapshots(owner); + ResourceOwnerEnlarge(owner); snap->regd_count++; ResourceOwnerRememberSnapshot(owner, snap); @@ -2345,3 +2363,19 @@ XidInMVCCSnapshot(TransactionId xid, Snapshot snapshot) return false; } + +/* + * ResourceOwner callbacks + */ +static void +ResOwnerReleaseSnapshot(Datum res) +{ + UnregisterSnapshot((Snapshot) DatumGetPointer(res)); +} + +static void +ResOwnerPrintSnapshotLeakWarning(Datum res) +{ + elog(WARNING, "Snapshot reference leak: Snapshot %p still referenced", + DatumGetPointer(res)); +} diff --git a/src/common/cryptohash_openssl.c b/src/common/cryptohash_openssl.c index 551ec392b60..6421f1de5cd 100644 --- a/src/common/cryptohash_openssl.c +++ b/src/common/cryptohash_openssl.c @@ -27,7 +27,6 @@ #ifndef FRONTEND #include "utils/memutils.h" #include "utils/resowner.h" -#include "utils/resowner_private.h" #endif /* @@ -60,6 +59,27 @@ struct pg_cryptohash_ctx #endif }; +/* ResourceOwner callbacks to hold cryptohash contexts */ +#ifndef FRONTEND +static void ResOwnerReleaseCryptoHash(Datum res); +static void ResOwnerPrintCryptoHashLeakWarning(Datum res); + +static ResourceOwnerFuncs cryptohash_resowner_funcs = +{ + /* relcache references */ + .name = "OpenSSL cryptohash context", + .phase = RESOURCE_RELEASE_BEFORE_LOCKS, + .ReleaseResource = ResOwnerReleaseCryptoHash, + .PrintLeakWarning = ResOwnerPrintCryptoHashLeakWarning, +}; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberCryptoHash(owner, ctx) \ + ResourceOwnerRemember(owner, PointerGetDatum(ctx), &cryptohash_resowner_funcs) +#define ResourceOwnerForgetCryptoHash(owner, ctx) \ + ResourceOwnerForget(owner, PointerGetDatum(ctx), &cryptohash_resowner_funcs) +#endif + /* * pg_cryptohash_create * @@ -77,7 +97,7 @@ pg_cryptohash_create(pg_cryptohash_type type) * allocation to avoid leaking. */ #ifndef FRONTEND - ResourceOwnerEnlargeCryptoHash(CurrentResourceOwner); + ResourceOwnerEnlarge(CurrentResourceOwner); #endif ctx = ALLOC(sizeof(pg_cryptohash_ctx)); @@ -106,8 +126,7 @@ pg_cryptohash_create(pg_cryptohash_type type) #ifndef FRONTEND ctx->resowner = CurrentResourceOwner; - ResourceOwnerRememberCryptoHash(CurrentResourceOwner, - PointerGetDatum(ctx)); + ResourceOwnerRememberCryptoHash(CurrentResourceOwner, ctx); #endif return ctx; @@ -207,10 +226,28 @@ pg_cryptohash_free(pg_cryptohash_ctx *ctx) EVP_MD_CTX_destroy(ctx->evpctx); #ifndef FRONTEND - ResourceOwnerForgetCryptoHash(ctx->resowner, - PointerGetDatum(ctx)); + ResourceOwnerForgetCryptoHash(ctx->resowner, ctx); #endif explicit_bzero(ctx, sizeof(pg_cryptohash_ctx)); FREE(ctx); } + + +/* + * ResourceOwner callbacks + */ +#ifndef FRONTEND +static void +ResOwnerReleaseCryptoHash(Datum res) +{ + pg_cryptohash_free((pg_cryptohash_ctx *) DatumGetPointer(res)); +} + +static void +ResOwnerPrintCryptoHashLeakWarning(Datum res) +{ + elog(WARNING, "cryptohash context reference leak: context %p still referenced", + DatumGetPointer(res)); +} +#endif diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index f6b57829653..6815dc38a16 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -296,6 +296,15 @@ typedef struct CkptSortItem extern CkptSortItem *CkptBufferIds; +/* ResourceOwner callbacks to hold buffer pins */ +extern ResourceOwnerFuncs buffer_resowner_funcs; + +/* Convenience wrappers over ResourceOwnerRemember/Forget */ +#define ResourceOwnerRememberBuffer(owner, buffer) \ + ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_resowner_funcs) +#define ResourceOwnerForgetBuffer(owner, buffer) \ + ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_resowner_funcs) + /* * Internal buffer management routines */ diff --git a/src/include/utils/catcache.h b/src/include/utils/catcache.h index ddc2762eb3f..2bf95c22e84 100644 --- a/src/include/utils/catcache.h +++ b/src/include/utils/catcache.h @@ -225,7 +225,4 @@ extern void PrepareToInvalidateCacheTuple(Relation relation, HeapTuple newtuple, void (*function) (int, uint32, Oid)); -extern void PrintCatCacheLeakWarning(HeapTuple tuple); -extern void PrintCatCacheListLeakWarning(CatCList *list); - #endif /* CATCACHE_H */ diff --git a/src/include/utils/plancache.h b/src/include/utils/plancache.h index 79d96e5ed03..964af79b30d 100644 --- a/src/include/utils/plancache.h +++ b/src/include/utils/plancache.h @@ -233,4 +233,6 @@ extern bool CachedPlanIsSimplyValid(CachedPlanSource *plansource, extern CachedExpression *GetCachedExpression(Node *expr); extern void FreeCachedExpression(CachedExpression *cexpr); +extern ResourceOwnerFuncs planref_resowner_funcs; + #endif /* PLANCACHE_H */ diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 109ac31b248..d59d14c0d01 100644 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -50,6 +50,36 @@ typedef enum RESOURCE_RELEASE_AFTER_LOCKS } ResourceReleasePhase; +/* + * In order to track an object, resowner.c needs a few callbacks for it. + * The callbacks for an resource of a specific kind are encapsulated in + * ResourceOwnerFuncs. + * + * Note that the callback occurs post-commit or post-abort, so these callback + * functions can only do noncritical cleanup. + */ +typedef struct ResourceOwnerFuncs +{ + const char *name; /* name for the object kind, for debugging */ + + ResourceReleasePhase phase; /* when are these objects released? */ + + /* + * Release resource. + * + * NOTE: this must call ResourceOwnerForget to disassociate it with the + * resource owner. + */ + void (*ReleaseResource)(Datum res); + + /* + * Print a warning, when a resource has not been properly released before + * commit. + */ + void (*PrintLeakWarning)(Datum res); + +} ResourceOwnerFuncs; + /* * Dynamically loaded modules can get control during ResourceOwnerRelease * by providing a callback of this form. @@ -71,16 +101,29 @@ extern void ResourceOwnerRelease(ResourceOwner owner, ResourceReleasePhase phase, bool isCommit, bool isTopLevel); -extern void ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner); extern void ResourceOwnerDelete(ResourceOwner owner); extern ResourceOwner ResourceOwnerGetParent(ResourceOwner owner); extern void ResourceOwnerNewParent(ResourceOwner owner, ResourceOwner newparent); + +extern void ResourceOwnerEnlarge(ResourceOwner owner); +extern void ResourceOwnerRemember(ResourceOwner owner, Datum res, ResourceOwnerFuncs *kind); +extern void ResourceOwnerForget(ResourceOwner owner, Datum res, ResourceOwnerFuncs *kind); + extern void RegisterResourceReleaseCallback(ResourceReleaseCallback callback, void *arg); extern void UnregisterResourceReleaseCallback(ResourceReleaseCallback callback, void *arg); + extern void CreateAuxProcessResourceOwner(void); extern void ReleaseAuxProcessResources(bool isCommit); +/* special support for local lock management */ +struct LOCALLOCK; +extern void ResourceOwnerRememberLock(ResourceOwner owner, struct LOCALLOCK *locallock); +extern void ResourceOwnerForgetLock(ResourceOwner owner, struct LOCALLOCK *locallock); + +/* special function to relase all plancache references */ +extern void ResourceOwnerReleaseAllPlanCacheRefs(ResourceOwner owner); + #endif /* RESOWNER_H */ diff --git a/src/include/utils/resowner_private.h b/src/include/utils/resowner_private.h deleted file mode 100644 index c480a1a24be..00000000000 --- a/src/include/utils/resowner_private.h +++ /dev/null @@ -1,105 +0,0 @@ -/*------------------------------------------------------------------------- - * - * resowner_private.h - * POSTGRES resource owner private definitions. - * - * See utils/resowner/README for more info. - * - * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/utils/resowner_private.h - * - *------------------------------------------------------------------------- - */ -#ifndef RESOWNER_PRIVATE_H -#define RESOWNER_PRIVATE_H - -#include "storage/dsm.h" -#include "storage/fd.h" -#include "storage/lock.h" -#include "utils/catcache.h" -#include "utils/plancache.h" -#include "utils/resowner.h" -#include "utils/snapshot.h" - - -/* support for buffer refcount management */ -extern void ResourceOwnerEnlargeBuffers(ResourceOwner owner); -extern void ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer); -extern void ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer); - -/* support for local lock management */ -extern void ResourceOwnerRememberLock(ResourceOwner owner, LOCALLOCK *locallock); -extern void ResourceOwnerForgetLock(ResourceOwner owner, LOCALLOCK *locallock); - -/* support for catcache refcount management */ -extern void ResourceOwnerEnlargeCatCacheRefs(ResourceOwner owner); -extern void ResourceOwnerRememberCatCacheRef(ResourceOwner owner, - HeapTuple tuple); -extern void ResourceOwnerForgetCatCacheRef(ResourceOwner owner, - HeapTuple tuple); -extern void ResourceOwnerEnlargeCatCacheListRefs(ResourceOwner owner); -extern void ResourceOwnerRememberCatCacheListRef(ResourceOwner owner, - CatCList *list); -extern void ResourceOwnerForgetCatCacheListRef(ResourceOwner owner, - CatCList *list); - -/* support for relcache refcount management */ -extern void ResourceOwnerEnlargeRelationRefs(ResourceOwner owner); -extern void ResourceOwnerRememberRelationRef(ResourceOwner owner, - Relation rel); -extern void ResourceOwnerForgetRelationRef(ResourceOwner owner, - Relation rel); - -/* support for plancache refcount management */ -extern void ResourceOwnerEnlargePlanCacheRefs(ResourceOwner owner); -extern void ResourceOwnerRememberPlanCacheRef(ResourceOwner owner, - CachedPlan *plan); -extern void ResourceOwnerForgetPlanCacheRef(ResourceOwner owner, - CachedPlan *plan); - -/* support for tupledesc refcount management */ -extern void ResourceOwnerEnlargeTupleDescs(ResourceOwner owner); -extern void ResourceOwnerRememberTupleDesc(ResourceOwner owner, - TupleDesc tupdesc); -extern void ResourceOwnerForgetTupleDesc(ResourceOwner owner, - TupleDesc tupdesc); - -/* support for snapshot refcount management */ -extern void ResourceOwnerEnlargeSnapshots(ResourceOwner owner); -extern void ResourceOwnerRememberSnapshot(ResourceOwner owner, - Snapshot snapshot); -extern void ResourceOwnerForgetSnapshot(ResourceOwner owner, - Snapshot snapshot); - -/* support for temporary file management */ -extern void ResourceOwnerEnlargeFiles(ResourceOwner owner); -extern void ResourceOwnerRememberFile(ResourceOwner owner, - File file); -extern void ResourceOwnerForgetFile(ResourceOwner owner, - File file); - -/* support for dynamic shared memory management */ -extern void ResourceOwnerEnlargeDSMs(ResourceOwner owner); -extern void ResourceOwnerRememberDSM(ResourceOwner owner, - dsm_segment *); -extern void ResourceOwnerForgetDSM(ResourceOwner owner, - dsm_segment *); - -/* support for JITContext management */ -extern void ResourceOwnerEnlargeJIT(ResourceOwner owner); -extern void ResourceOwnerRememberJIT(ResourceOwner owner, - Datum handle); -extern void ResourceOwnerForgetJIT(ResourceOwner owner, - Datum handle); - -/* support for cryptohash context management */ -extern void ResourceOwnerEnlargeCryptoHash(ResourceOwner owner); -extern void ResourceOwnerRememberCryptoHash(ResourceOwner owner, - Datum handle); -extern void ResourceOwnerForgetCryptoHash(ResourceOwner owner, - Datum handle); - -#endif /* RESOWNER_PRIVATE_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 943142ced8c..2df8d3cbefa 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2127,8 +2127,10 @@ ReplicationStateOnDisk ResTarget ReservoirState ReservoirStateData -ResourceArray +ResourceElem ResourceOwner +ResourceOwnerData +ResourceOwnerFuncs ResourceReleaseCallback ResourceReleaseCallbackItem ResourceReleasePhase -- 2.29.2 --------------F30A5443A5D524A5CBC51AE7 Content-Type: text/x-patch; charset=UTF-8; name="v6-0003-Optimize-hash-function.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v6-0003-Optimize-hash-function.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ec86a17b3b3..64772877b0f 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 37b150b44ba..c99edae575b 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 8a560903ede..c1e9fe366f3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -402,6 +407,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v21-0001-Add-CopyToRountine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). --- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index cfc74ee7b1..b5d8678394 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -69,7 +69,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -117,7 +116,7 @@ CopyToTextSendEndOfRow(CopyToState cstate) default: break; } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -302,7 +301,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } static void @@ -311,7 +310,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } CopyToRoutine CopyToRoutineText = { @@ -377,8 +376,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -401,8 +400,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index a869d78d72..ffad433a21 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -174,6 +174,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.43.0 ----Next_Part(Wed_Jan_24_14_49_36_2024_411)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v6-0005-Extract-COPY-FROM-format-implementations.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 74844103228..a115d7f9e26 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 54aa6cdecaf..b8d0e996117 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -500,6 +500,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index e298b19860c..5665408eaa0 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -402,6 +407,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Jul_30_16_13_06_2024_473)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf * Rename CopySendEndOfRow() to CopyToStateFlush() because it's a method for CopyToState and it's used for flushing. End-of-row related codes were moved to CopyToTextSendEndOfRow(). For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/backend/commands/copyto.c | 15 +++++++-------- src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 74844103228..cd80d34f3da 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -164,7 +164,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -193,7 +192,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -221,7 +220,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -363,7 +362,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -380,7 +379,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -691,14 +690,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -1078,7 +1077,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -2103,8 +2102,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 54aa6cdecaf..cd9e352533a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -63,7 +63,6 @@ static void SendCopyEnd(CopyToState cstate); static void CopySendData(CopyToState cstate, const void *databuf, int datasize); static void CopySendString(CopyToState cstate, const char *str); static void CopySendChar(CopyToState cstate, char c); -static void CopySendEndOfRow(CopyToState cstate); static void CopySendInt32(CopyToState cstate, int32 val); static void CopySendInt16(CopyToState cstate, int16 val); @@ -99,7 +98,7 @@ CopyToTextLikeSendEndOfRow(CopyToState cstate) } /* Now take the actions related to the end of a row */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -325,7 +324,7 @@ CopyToBinaryOneRow(CopyToState cstate, TupleTableSlot *slot) } } - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -339,7 +338,7 @@ CopyToBinaryEnd(CopyToState cstate) /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); /* Need to flush out the trailer */ - CopySendEndOfRow(cstate); + CopyToStateFlush(cstate); } /* @@ -419,8 +418,8 @@ SendCopyEnd(CopyToState cstate) * CopySendData sends output data to the destination (file or frontend) * CopySendString does the same for null-terminated strings * CopySendChar does the same for single characters - * CopySendEndOfRow does the appropriate thing at end of each data row - * (data is not actually flushed except by CopySendEndOfRow) + * CopyToStateFlush flushes the buffered data + * (data is not actually flushed except by CopyToStateFlush) * * NB: no data conversion is applied by these functions *---------- @@ -443,8 +442,8 @@ CopySendChar(CopyToState cstate, char c) appendStringInfoCharMacro(cstate->fe_msgbuf, c); } -static void -CopySendEndOfRow(CopyToState cstate) +void +CopyToStateFlush(CopyToState cstate) { StringInfo fe_msgbuf = cstate->fe_msgbuf; diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 3104d99ea9f..0820b47a2d2 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -401,6 +406,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Wed_Jul_24_17_30_59_2024_070)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension @ 2024-01-23 06:12 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-23 06:12 UTC (permalink / raw) For custom COPY TO format implementation: * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() For custom COPY FROM format implementation: * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 10 ++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ccfbacb4a37..4fa23d992f5 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index fb68f42ce1e..93b041352c5 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 206d4c9fac9..2de610ef729 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -302,8 +302,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + typedef struct CopyToStateData *CopyToState; @@ -405,6 +410,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v23-0001-Add-CopyToRountine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension @ 2024-01-24 05:19 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-01-24 05:19 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data * Rename CopyReadBinaryData() to CopyFromStateRead() because it's a method for CopyFromState and "BinaryData" is redundant. --- src/backend/commands/copyfromparse.c | 21 ++++++++++----------- src/include/commands/copyapi.h | 5 +++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index a78a790060..f8a194635d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -165,7 +165,6 @@ static int CopyGetData(CopyFromState cstate, void *databuf, static inline bool CopyGetInt32(CopyFromState cstate, int32 *val); static inline bool CopyGetInt16(CopyFromState cstate, int16 *val); static void CopyLoadInputBuf(CopyFromState cstate); -static int CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes); void ReceiveCopyBegin(CopyFromState cstate) @@ -194,7 +193,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) int32 tmp; /* Signature */ - if (CopyReadBinaryData(cstate, readSig, 11) != 11 || + if (CopyFromStateRead(cstate, readSig, 11) != 11 || memcmp(readSig, BinarySignature, 11) != 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), @@ -222,7 +221,7 @@ ReceiveCopyBinaryHeader(CopyFromState cstate) /* Skip extension header, if present */ while (tmp-- > 0) { - if (CopyReadBinaryData(cstate, readSig, 1) != 1) + if (CopyFromStateRead(cstate, readSig, 1) != 1) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("invalid COPY file header (wrong length)"))); @@ -364,7 +363,7 @@ CopyGetInt32(CopyFromState cstate, int32 *val) { uint32 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -381,7 +380,7 @@ CopyGetInt16(CopyFromState cstate, int16 *val) { uint16 buf; - if (CopyReadBinaryData(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) + if (CopyFromStateRead(cstate, (char *) &buf, sizeof(buf)) != sizeof(buf)) { *val = 0; /* suppress compiler warning */ return false; @@ -692,14 +691,14 @@ CopyLoadInputBuf(CopyFromState cstate) } /* - * CopyReadBinaryData + * CopyFromStateRead * * Reads up to 'nbytes' bytes from cstate->copy_file via cstate->raw_buf * and writes them to 'dest'. Returns the number of bytes read (which * would be less than 'nbytes' only if we reach EOF). */ -static int -CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) { int copied_bytes = 0; @@ -988,7 +987,7 @@ CopyFromBinaryOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, */ char dummy; - if (CopyReadBinaryData(cstate, &dummy, 1) > 0) + if (CopyFromStateRead(cstate, &dummy, 1) > 0) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("received copy data after EOF marker"))); @@ -1997,8 +1996,8 @@ CopyReadBinaryAttribute(CopyFromState cstate, FmgrInfo *flinfo, resetStringInfo(&cstate->attribute_buf); enlargeStringInfo(&cstate->attribute_buf, fld_size); - if (CopyReadBinaryData(cstate, cstate->attribute_buf.data, - fld_size) != fld_size) + if (CopyFromStateRead(cstate, cstate->attribute_buf.data, + fld_size) != fld_size) ereport(ERROR, (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), errmsg("unexpected EOF in COPY data"))); diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b7e8f627bf..22accc83ab 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -314,8 +314,13 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + /* * Represents the different dest cases we need to worry about at * the bottom level -- 2.43.0 ----Next_Part(Wed_Jan_24_14_49_36_2024_411)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension @ 2024-09-28 14:59 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index fb68f42ce1e..93b041352c5 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index b6ddb5f6216..310a37ba728 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -190,6 +190,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v23-0006-Add-CopyFromRoutine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension @ 2024-09-28 14:59 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 14:59 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 37b150b44ba..c99edae575b 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -496,6 +496,20 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * CopyToStateFlush + * + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * These functions do apply some data conversion */ diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 03779c15f43..30765951e2e 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -187,6 +187,11 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v21-0006-Add-CopyFromRoutine.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension @ 2024-09-28 15:32 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ccfbacb4a37..4fa23d992f5 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 0274e3487c3..2de610ef729 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -302,8 +302,14 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + + typedef struct CopyToStateData *CopyToState; /* -- 2.45.2 ----Next_Part(Tue_Nov__5_17_43_28_2024_751)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension @ 2024-09-28 15:32 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-09-28 15:32 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 14 ++++++++++++++ src/include/commands/copyapi.h | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index ec86a17b3b3..64772877b0f 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,20 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * CopyFromStateRead + * + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c118558ee71..c1e9fe366f3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -299,8 +299,14 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + + typedef struct CopyToStateData *CopyToState; /* -- 2.45.2 ----Next_Part(Sun_Sep_29_00_56_45_2024_080)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 96b5e144a1d..cb9bfa0053f 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.45.2 ----Next_Part(Wed_Nov_27_16_53_44_2024_871)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v27-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Wed_Mar_19_11_56_17_2025_532)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v37-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 91fa46ddf6f..da281f32950 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 5d071b378d6..f8167af4c79 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 2df53dda8a0..4b82372691e 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts); -- 2.47.1 ----Next_Part(Fri_Jan_31_00_42_13_2025_303)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v29-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 17d89c23af0..35f9035141a 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Sat_Mar__1_11_50_09_2025_878)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v35-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Thu_Mar_20_10_24_55_2025_309)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v38-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 16d3b389e97..20d49d73e38 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c125dc3e209..d0da9e07a0d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.1 ----Next_Part(Thu_Feb__6_21_06_31_2025_538)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v32-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 1e75e07dc0b..7487190bdd6 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.47.1 ----Next_Part(Thu_Jan_23_18_12_10_2025_763)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v28-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 91fa46ddf6f..da281f32950 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 5d071b378d6..f8167af4c79 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 2df53dda8a0..4b82372691e 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; const struct CopyToRoutine *CopyToGetBuiltinRoutine(CopyFormatOptions *opts); -- 2.47.1 ----Next_Part(Sat_Feb__1_08_10_23_2025_403)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v30-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 99c2f2dd699..f5ed3efbace 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -458,6 +458,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 4f4ffabf882..5c5ea6592e3 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.2 ----Next_Part(Wed_Mar__5_09_06_08_2025_350)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v36-0004-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 96b5e144a1d..cb9bfa0053f 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -442,6 +442,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 1cb2815deab..030a82aca7f 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -116,8 +116,13 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. -- 2.45.2 ----Next_Part(Mon_Nov_25_15_01_50_2024_156)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v26-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension @ 2024-11-25 05:01 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:01 UTC (permalink / raw) * Add CopyToStateData::opaque that can be used to keep data for custom COPY TO format implementation * Export CopySendEndOfRow() to flush data in CopyToStateData::fe_msgbuf as CopyToStateFlush() --- src/backend/commands/copyto.c | 12 ++++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyto_internal.h | 3 +++ 3 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 16d3b389e97..20d49d73e38 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -461,6 +461,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * the line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index c125dc3e209..d0da9e07a0d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -54,6 +54,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 1b58b36c0a3..ce1c33a4004 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ -- 2.47.1 ----Next_Part(Sat_Feb__1_19_12_01_2025_760)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v31-0006-Add-support-for-adding-custom-COPY-FROM-format.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index f7982bf692f..650b6b2382b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index bf933069fea..d1a1dbeb178 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -105,4 +105,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3743b11faa4..a65bbbc962e 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Sat_Feb__1_08_10_23_2025_403)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v30-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 75b49629f08..01f2e7a8824 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 103eb21767d..ac58adbd23d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -104,4 +104,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Sat_Feb__1_19_12_01_2025_760)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v31-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 75b49629f08..01f2e7a8824 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -730,6 +730,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 103eb21767d..ac58adbd23d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -104,4 +104,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Thu_Feb__6_21_06_31_2025_538)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v32-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Sat_Mar__1_11_50_09_2025_878)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v35-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Wed_Mar__5_09_06_08_2025_350)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v36-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Wed_Mar_19_11_56_17_2025_532)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v37-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..d8fd238e72b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 895c105d8d8..2044d8b8c4c 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -108,4 +108,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.2 ----Next_Part(Thu_Mar_20_10_24_55_2025_309)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v38-0007-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyGetData() to get the next data as CopyFromStateGetData() --- src/backend/commands/copyfromparse.c | 11 +++++++++++ src/include/commands/copyapi.h | 2 ++ src/include/commands/copyfrom_internal.h | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index f7982bf692f..650b6b2382b 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index bf933069fea..d1a1dbeb178 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -105,4 +105,6 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3743b11faa4..a65bbbc962e 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); -- 2.47.1 ----Next_Part(Fri_Jan_31_00_42_13_2025_303)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v29-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 1c68b0d2952..0a7e7255b7d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 335584f8877..caba308533d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Mon_Nov_25_15_01_50_2024_156)---- ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 1c68b0d2952..0a7e7255b7d 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 335584f8877..caba308533d 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.45.2 ----Next_Part(Wed_Nov_27_16_53_44_2024_871)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v27-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension @ 2024-11-25 05:21 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2024-11-25 05:21 UTC (permalink / raw) * Add CopyFromStateData::opaque that can be used to keep data for custom COPY From format implementation * Export CopyReadBinaryData() to read the next data as CopyFromStateRead() --- src/backend/commands/copyfromparse.c | 12 ++++++++++++ src/include/commands/copyapi.h | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 5fcdbea2c2a..d79b6ebe8a3 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -729,6 +729,18 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyReadBinaryData() for extensions. We want to keep + * CopyReadBinaryData() as a static function for + * optimization. CopyReadBinaryData() calls in this file may be optimized by + * a compiler. + */ +int +CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes) +{ + return CopyReadBinaryData(cstate, dest, nbytes); +} + /* * Read raw fields in the next line for COPY FROM in text or csv mode. * Return false if no more lines. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 9358515c6f6..6f158272829 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -335,6 +335,11 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; +extern int CopyFromStateRead(CopyFromState cstate, char *dest, int nbytes); + #endif /* COPYAPI_H */ -- 2.47.1 ----Next_Part(Thu_Jan_23_18_12_10_2025_763)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v28-0009-Add-CopyFromSkipErrorRow-for-custom-COPY-format-.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v39 3/5] Add support for implementing custom COPY handler as extension @ 2025-03-27 02:24 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw) * TO: Add CopyToStateData::opaque that can be used to keep data for custom COPY TO handler implementation * TO: Export CopySendEndOfRow() to send end of row data as CopyToStateFlush() * FROM: Add CopyFromStateData::opaque that can be used to keep data for custom COPY FROM handler implementation * FROM: Export CopyGetData() to get the next data as CopyFromStateGetData() * FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and "LOG_VERBOSITY verbose" COPY FROM extensions must call CopyFromSkipErrorRow() when CopyFromOneRow callback reports an error by errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases. --- src/backend/commands/copyfromparse.c | 93 ++++++++++++------- src/backend/commands/copyto.c | 12 +++ src/include/commands/copyapi.h | 6 ++ src/include/commands/copyfrom_internal.h | 3 + src/include/commands/copyto_internal.h | 3 + .../test_copy_format/expected/no_schema.out | 47 ++++++++++ .../test_copy_format/sql/no_schema.sql | 24 +++++ .../test_copy_format/test_copy_format.c | 80 +++++++++++++++- 8 files changed, 231 insertions(+), 37 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 17e51f02e04..2070f51a963 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. @@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true); } +/* + * Call this when you report an error by errsave() in your CopyFromOneRow + * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases + * for you. + */ +void +CopyFromSkipErrorRow(CopyFromState cstate) +{ + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + + cstate->num_errors++; + + if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) + { + /* + * Since we emit line number and column info in the below notice + * message, we suppress error context information other than the + * relation name. + */ + Assert(!cstate->relname_only); + cstate->relname_only = true; + + if (cstate->cur_attval) + { + char *attval; + + attval = CopyLimitPrintoutLength(cstate->cur_attval); + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname, + attval)); + pfree(attval); + } + else + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input", + (unsigned long long) cstate->cur_lineno, + cstate->cur_attname)); + + /* reset relname_only */ + cstate->relname_only = false; + } +} + /* * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow(). * @@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { - Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); - - cstate->num_errors++; - - if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) - { - /* - * Since we emit line number and column info in the below - * notice message, we suppress error context information other - * than the relation name. - */ - Assert(!cstate->relname_only); - cstate->relname_only = true; - - if (cstate->cur_attval) - { - char *attval; - - attval = CopyLimitPrintoutLength(cstate->cur_attval); - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": \"%s\"", - (unsigned long long) cstate->cur_lineno, - cstate->cur_attname, - attval)); - pfree(attval); - } - else - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %llu for column \"%s\": null input", - (unsigned long long) cstate->cur_lineno, - cstate->cur_attname)); - - /* reset relname_only */ - cstate->relname_only = false; - } - + CopyFromSkipErrorRow(cstate); return true; } diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index b7ff6466ce3..23cbdad184c 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -456,6 +456,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 53ad3337f86..500ece7d5bb 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. @@ -106,4 +108,8 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + +extern void CopyFromSkipErrorRow(CopyFromState cstate); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 3a306e3286e..af425cf5fd9 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index 12c4a0f5979..14ee0f50588 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ diff --git a/src/test/modules/test_copy_format/expected/no_schema.out b/src/test/modules/test_copy_format/expected/no_schema.out index d5903632b2e..05d160c1eae 100644 --- a/src/test/modules/test_copy_format/expected/no_schema.out +++ b/src/test/modules/test_copy_format/expected/no_schema.out @@ -1,6 +1,8 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=true NOTICE: CopyFromInFunc: attribute: smallint @@ -8,7 +10,50 @@ NOTICE: CopyFromInFunc: attribute: integer NOTICE: CopyFromInFunc: attribute: bigint NOTICE: CopyFromStart: the number of attributes: 3 NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: invalid value: "6" +CONTEXT: COPY test, line 2, column a: "6" +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: skipping row due to data type incompatibility at line 2 for column "a": "6" +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility +NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: too much lines: 3 +CONTEXT: COPY test, line 3 COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=false NOTICE: CopyToOutFunc: attribute: smallint @@ -18,6 +63,8 @@ NOTICE: CopyToStart: the number of attributes: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToEnd DROP TABLE public.test; DROP EXTENSION test_copy_format; diff --git a/src/test/modules/test_copy_format/sql/no_schema.sql b/src/test/modules/test_copy_format/sql/no_schema.sql index 1e049f799f0..1901c4a9f43 100644 --- a/src/test/modules/test_copy_format/sql/no_schema.sql +++ b/src/test/modules/test_copy_format/sql/no_schema.sql @@ -1,7 +1,31 @@ CREATE EXTENSION test_copy_format; CREATE TABLE public.test (a smallint, b integer, c bigint); INSERT INTO public.test VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY public.test FROM stdin WITH (FORMAT 'test_copy_format'); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +987 +654 +\. +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY public.test FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +321 \. COPY public.test TO stdout WITH (FORMAT 'test_copy_format'); DROP TABLE public.test; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 1d754201336..34ec693a7ec 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "commands/copyapi.h" +#include "commands/copyfrom_internal.h" #include "commands/defrem.h" #include "utils/builtins.h" @@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) static bool TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { + int n_attributes = list_length(cstate->attnumlist); + char *line; + int line_size = n_attributes + 1; /* +1 is for new line */ + int read_bytes; + ereport(NOTICE, (errmsg("CopyFromOneRow"))); - return false; + + cstate->cur_lineno++; + line = palloc(line_size); + read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size); + if (read_bytes == 0) + return false; + if (read_bytes != line_size) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("one line must be %d bytes: %d", + line_size, read_bytes))); + + if (cstate->cur_lineno == 1) + { + /* Success */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + ListCell *cur; + int i = 0; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (att->atttypid == INT2OID) + { + values[i] = Int16GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT4OID) + { + values[i] = Int32GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT8OID) + { + values[i] = Int64GetDatum(line[i] - '0'); + } + nulls[i] = false; + i++; + } + } + else if (cstate->cur_lineno == 2) + { + /* Soft error */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + int attnum = lfirst_int(list_head(cstate->attnumlist)); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + char value[2]; + + cstate->cur_attname = NameStr(att->attname); + value[0] = line[0]; + value[1] = '\0'; + cstate->cur_attval = value; + errsave((Node *) cstate->escontext, + ( + errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid value: \"%c\"", line[0]))); + CopyFromSkipErrorRow(cstate); + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; + return true; + } + else + { + /* Hard error */ + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("too much lines: %llu", + (unsigned long long) cstate->cur_lineno))); + } + + return true; } static void -- 2.47.2 ----Next_Part(Thu_Mar_27_12_28_40_2025_510)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v39-0004-Use-copy-handlers-for-built-in-formats.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
* [PATCH v40 3/6] Add support for implementing custom COPY handler as extension @ 2025-03-27 02:24 Sutou Kouhei <[email protected]> 0 siblings, 0 replies; 35+ messages in thread From: Sutou Kouhei @ 2025-03-27 02:24 UTC (permalink / raw) * TO: Add CopyToStateData::opaque that can be used to keep data for custom COPY TO handler implementation * TO: Export CopySendEndOfRow() to send end of row data as CopyToStateFlush() * FROM: Add CopyFromStateData::opaque that can be used to keep data for custom COPY FROM handler implementation * FROM: Export CopyGetData() to get the next data as CopyFromStateGetData() * FROM: Add CopyFromSkipErrorRow() for "ON_ERROR stop" and "LOG_VERBOSITY verbose" COPY FROM extensions must call CopyFromSkipErrorRow() when CopyFromOneRow callback reports an error by errsave(). CopyFromSkipErrorRow() handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases. --- src/backend/commands/copyfromparse.c | 93 ++++++++++++------- src/backend/commands/copyto.c | 12 +++ src/include/commands/copyapi.h | 6 ++ src/include/commands/copyfrom_internal.h | 3 + src/include/commands/copyto_internal.h | 3 + .../expected/test_copy_format.out | 50 ++++++++++ .../test_copy_format/sql/test_copy_format.sql | 35 +++++++ .../test_copy_format/test_copy_format.c | 80 +++++++++++++++- 8 files changed, 245 insertions(+), 37 deletions(-) diff --git a/src/backend/commands/copyfromparse.c b/src/backend/commands/copyfromparse.c index 9f7171d1478..de68b53b000 100644 --- a/src/backend/commands/copyfromparse.c +++ b/src/backend/commands/copyfromparse.c @@ -739,6 +739,17 @@ CopyReadBinaryData(CopyFromState cstate, char *dest, int nbytes) return copied_bytes; } +/* + * Export CopyGetData() for extensions. We want to keep CopyGetData() as a + * static function for optimization. CopyGetData() calls in this file may be + * optimized by a compiler. + */ +int +CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread) +{ + return CopyGetData(cstate, dest, minread, maxread); +} + /* * This function is exposed for use by extensions that read raw fields in the * next line. See NextCopyFromRawFieldsInternal() for details. @@ -927,6 +938,51 @@ CopyFromCSVOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, return CopyFromTextLikeOneRow(cstate, econtext, values, nulls, true); } +/* + * Call this when you report an error by errsave() in your CopyFromOneRow + * callback. This handles "ON_ERROR stop" and "LOG_VERBOSITY verbose" cases + * for you. + */ +void +CopyFromSkipErrorRow(CopyFromState cstate) +{ + Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); + + cstate->num_errors++; + + if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) + { + /* + * Since we emit line number and column info in the below notice + * message, we suppress error context information other than the + * relation name. + */ + Assert(!cstate->relname_only); + cstate->relname_only = true; + + if (cstate->cur_attval) + { + char *attval; + + attval = CopyLimitPrintoutLength(cstate->cur_attval); + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", + cstate->cur_lineno, + cstate->cur_attname, + attval)); + pfree(attval); + } + else + ereport(NOTICE, + errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", + cstate->cur_lineno, + cstate->cur_attname)); + + /* reset relname_only */ + cstate->relname_only = false; + } +} + /* * Workhorse for CopyFromTextOneRow() and CopyFromCSVOneRow(). * @@ -1033,42 +1089,7 @@ CopyFromTextLikeOneRow(CopyFromState cstate, ExprContext *econtext, (Node *) cstate->escontext, &values[m])) { - Assert(cstate->opts.on_error != COPY_ON_ERROR_STOP); - - cstate->num_errors++; - - if (cstate->opts.log_verbosity == COPY_LOG_VERBOSITY_VERBOSE) - { - /* - * Since we emit line number and column info in the below - * notice message, we suppress error context information other - * than the relation name. - */ - Assert(!cstate->relname_only); - cstate->relname_only = true; - - if (cstate->cur_attval) - { - char *attval; - - attval = CopyLimitPrintoutLength(cstate->cur_attval); - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": \"%s\"", - cstate->cur_lineno, - cstate->cur_attname, - attval)); - pfree(attval); - } - else - ereport(NOTICE, - errmsg("skipping row due to data type incompatibility at line %" PRIu64 " for column \"%s\": null input", - cstate->cur_lineno, - cstate->cur_attname)); - - /* reset relname_only */ - cstate->relname_only = false; - } - + CopyFromSkipErrorRow(cstate); return true; } diff --git a/src/backend/commands/copyto.c b/src/backend/commands/copyto.c index 265b847e255..d6fcfdfb9b1 100644 --- a/src/backend/commands/copyto.c +++ b/src/backend/commands/copyto.c @@ -454,6 +454,18 @@ CopySendEndOfRow(CopyToState cstate) resetStringInfo(fe_msgbuf); } +/* + * Export CopySendEndOfRow() for extensions. We want to keep + * CopySendEndOfRow() as a static function for + * optimization. CopySendEndOfRow() calls in this file may be optimized by a + * compiler. + */ +void +CopyToStateFlush(CopyToState cstate) +{ + CopySendEndOfRow(cstate); +} + /* * Wrapper function of CopySendEndOfRow for text and CSV formats. Sends the * line termination and do common appropriate things for the end of row. diff --git a/src/include/commands/copyapi.h b/src/include/commands/copyapi.h index 53ad3337f86..500ece7d5bb 100644 --- a/src/include/commands/copyapi.h +++ b/src/include/commands/copyapi.h @@ -56,6 +56,8 @@ typedef struct CopyToRoutine void (*CopyToEnd) (CopyToState cstate); } CopyToRoutine; +extern void CopyToStateFlush(CopyToState cstate); + /* * API structure for a COPY FROM format implementation. Note this must be * allocated in a server-lifetime manner, typically as a static const struct. @@ -106,4 +108,8 @@ typedef struct CopyFromRoutine void (*CopyFromEnd) (CopyFromState cstate); } CopyFromRoutine; +extern int CopyFromStateGetData(CopyFromState cstate, void *dest, int minread, int maxread); + +extern void CopyFromSkipErrorRow(CopyFromState cstate); + #endif /* COPYAPI_H */ diff --git a/src/include/commands/copyfrom_internal.h b/src/include/commands/copyfrom_internal.h index 24157e11a73..f9e27152313 100644 --- a/src/include/commands/copyfrom_internal.h +++ b/src/include/commands/copyfrom_internal.h @@ -181,6 +181,9 @@ typedef struct CopyFromStateData #define RAW_BUF_BYTES(cstate) ((cstate)->raw_buf_len - (cstate)->raw_buf_index) uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyFromStateData; extern void ReceiveCopyBegin(CopyFromState cstate); diff --git a/src/include/commands/copyto_internal.h b/src/include/commands/copyto_internal.h index da796131988..3bd9d702bf0 100644 --- a/src/include/commands/copyto_internal.h +++ b/src/include/commands/copyto_internal.h @@ -78,6 +78,9 @@ typedef struct CopyToStateData FmgrInfo *out_functions; /* lookup info for output functions */ MemoryContext rowcontext; /* per-row evaluation context */ uint64 bytes_processed; /* number of bytes processed so far */ + + /* For custom format implementation */ + void *opaque; /* private space */ } CopyToStateData; #endif /* COPYTO_INTERNAL_H */ diff --git a/src/test/modules/test_copy_format/expected/test_copy_format.out b/src/test/modules/test_copy_format/expected/test_copy_format.out index 3916b766615..47a875f0ab1 100644 --- a/src/test/modules/test_copy_format/expected/test_copy_format.out +++ b/src/test/modules/test_copy_format/expected/test_copy_format.out @@ -4,6 +4,8 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); -- schema. CREATE EXTENSION test_copy_format; -- We can find a custom COPY handler without schema. +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=true NOTICE: CopyFromInFunc: attribute: smallint @@ -11,7 +13,50 @@ NOTICE: CopyFromInFunc: attribute: integer NOTICE: CopyFromInFunc: attribute: bigint NOTICE: CopyFromStart: the number of attributes: 3 NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: invalid value: "6" +CONTEXT: COPY copy_data, line 2, column a: "6" +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: skipping row due to data type incompatibility at line 2 for column "a": "6" +NOTICE: CopyFromOneRow +NOTICE: 1 row was skipped due to data type incompatibility +NOTICE: CopyFromEnd +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +NOTICE: test_copy_format: is_from=true +NOTICE: CopyFromInFunc: attribute: smallint +NOTICE: CopyFromInFunc: attribute: integer +NOTICE: CopyFromInFunc: attribute: bigint +NOTICE: CopyFromStart: the number of attributes: 3 +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +NOTICE: CopyFromOneRow +ERROR: too much lines: 3 +CONTEXT: COPY copy_data, line 3 COPY copy_data TO stdout WITH (FORMAT 'test_copy_format'); NOTICE: test_copy_format: is_from=false NOTICE: CopyToOutFunc: attribute: smallint @@ -21,7 +66,12 @@ NOTICE: CopyToStart: the number of attributes: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 +NOTICE: CopyToOneRow: the number of valid values: 3 NOTICE: CopyToEnd +-- Reset data. +TRUNCATE copy_data; +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); DROP EXTENSION test_copy_format; -- Install custom COPY handlers to a schema that isn't included in -- search_path. diff --git a/src/test/modules/test_copy_format/sql/test_copy_format.sql b/src/test/modules/test_copy_format/sql/test_copy_format.sql index b262794f878..c7beb2fb8ae 100644 --- a/src/test/modules/test_copy_format/sql/test_copy_format.sql +++ b/src/test/modules/test_copy_format/sql/test_copy_format.sql @@ -4,10 +4,45 @@ INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); -- No WITH SCHEMA. It installs custom COPY handlers to the current -- schema. CREATE EXTENSION test_copy_format; + -- We can find a custom COPY handler without schema. + +-- 987 is accepted. +-- 654 is a hard error because ON_ERROR is stop by default. COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format'); +987 +654 \. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +\. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore, LOG_VERBOSITY verbose); +987 +654 +\. + +-- 987 is accepted. +-- 654 is a soft error because ON_ERROR is ignore. +-- 321 is a hard error. +COPY copy_data FROM stdin WITH (FORMAT 'test_copy_format', ON_ERROR ignore); +987 +654 +321 +\. + COPY copy_data TO stdout WITH (FORMAT 'test_copy_format'); + +-- Reset data. +TRUNCATE copy_data; +INSERT INTO copy_data VALUES (1, 2, 3), (12, 34, 56), (123, 456, 789); + DROP EXTENSION test_copy_format; diff --git a/src/test/modules/test_copy_format/test_copy_format.c b/src/test/modules/test_copy_format/test_copy_format.c index 1d754201336..34ec693a7ec 100644 --- a/src/test/modules/test_copy_format/test_copy_format.c +++ b/src/test/modules/test_copy_format/test_copy_format.c @@ -14,6 +14,7 @@ #include "postgres.h" #include "commands/copyapi.h" +#include "commands/copyfrom_internal.h" #include "commands/defrem.h" #include "utils/builtins.h" @@ -35,8 +36,85 @@ TestCopyFromStart(CopyFromState cstate, TupleDesc tupDesc) static bool TestCopyFromOneRow(CopyFromState cstate, ExprContext *econtext, Datum *values, bool *nulls) { + int n_attributes = list_length(cstate->attnumlist); + char *line; + int line_size = n_attributes + 1; /* +1 is for new line */ + int read_bytes; + ereport(NOTICE, (errmsg("CopyFromOneRow"))); - return false; + + cstate->cur_lineno++; + line = palloc(line_size); + read_bytes = CopyFromStateGetData(cstate, line, line_size, line_size); + if (read_bytes == 0) + return false; + if (read_bytes != line_size) + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("one line must be %d bytes: %d", + line_size, read_bytes))); + + if (cstate->cur_lineno == 1) + { + /* Success */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + ListCell *cur; + int i = 0; + + foreach(cur, cstate->attnumlist) + { + int attnum = lfirst_int(cur); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + + if (att->atttypid == INT2OID) + { + values[i] = Int16GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT4OID) + { + values[i] = Int32GetDatum(line[i] - '0'); + } + else if (att->atttypid == INT8OID) + { + values[i] = Int64GetDatum(line[i] - '0'); + } + nulls[i] = false; + i++; + } + } + else if (cstate->cur_lineno == 2) + { + /* Soft error */ + TupleDesc tupDesc = RelationGetDescr(cstate->rel); + int attnum = lfirst_int(list_head(cstate->attnumlist)); + int m = attnum - 1; + Form_pg_attribute att = TupleDescAttr(tupDesc, m); + char value[2]; + + cstate->cur_attname = NameStr(att->attname); + value[0] = line[0]; + value[1] = '\0'; + cstate->cur_attval = value; + errsave((Node *) cstate->escontext, + ( + errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid value: \"%c\"", line[0]))); + CopyFromSkipErrorRow(cstate); + cstate->cur_attname = NULL; + cstate->cur_attval = NULL; + return true; + } + else + { + /* Hard error */ + ereport(ERROR, + (errcode(ERRCODE_BAD_COPY_FILE_FORMAT), + errmsg("too much lines: %llu", + (unsigned long long) cstate->cur_lineno))); + } + + return true; } static void -- 2.47.2 ----Next_Part(Fri_Apr_25_21_45_34_2025_836)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v40-0004-Use-copy-handlers-for-built-in-formats.patch" ^ permalink raw reply [nested|flat] 35+ messages in thread
end of thread, other threads:[~2025-03-27 02:24 UTC | newest] Thread overview: 35+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-01-20 21:16 [PATCH v6 2/3] Make resowners more easily extensible. Heikki Linnakangas <[email protected]> 2024-01-23 06:12 [PATCH v20 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v6 4/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v19 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v18 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-23 06:12 [PATCH v22 5/5] Add support for implementing custom COPY TO/FROM format as extension Sutou Kouhei <[email protected]> 2024-01-24 05:19 [PATCH v6 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-09-28 14:59 [PATCH v21 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-09-28 14:59 [PATCH v23 05/10] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-09-28 15:32 [PATCH v21 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-09-28 15:32 [PATCH v23 10/10] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v30 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v27 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v38 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v32 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v28 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v31 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v26 5/8] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v35 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v37 3/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v36 3/7] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:01 [PATCH v29 5/9] Add support for implementing custom COPY TO format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v31 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v36 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v37 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v28 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v35 6/7] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v29 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v38 6/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v27 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v32 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v30 8/9] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2024-11-25 05:21 [PATCH v26 8/8] Add support for implementing custom COPY FROM format as extension Sutou Kouhei <[email protected]> 2025-03-27 02:24 [PATCH v40 3/6] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]> 2025-03-27 02:24 [PATCH v39 3/5] Add support for implementing custom COPY handler as extension Sutou Kouhei <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox