($INBOX_DIR/description missing)help / color / mirror / Atom feed
[PATCH 1/5] Adjust inlining of some functions 20+ messages / 4 participants [nested] [flat]
* [PATCH 1/5] Adjust inlining of some functions @ 2019-06-28 07:29 Kyotaro Horiguchi <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Kyotaro Horiguchi @ 2019-06-28 07:29 UTC (permalink / raw) SearchCatCacheInternal code path is quite short and hot so that it doesn't accept additional cycles in the function. But changing inline attribute of SearchCatCacheInternal and CatalogCacheComputeHashValue makes SearchCatCacheN faster by about 6%. This makes room for an extra branch to be the door to other implementations of catcache. --- src/backend/utils/cache/catcache.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 00def27881..8fc067ce31 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -63,10 +63,10 @@ /* Cache management header --- pointer is NULL until created */ static CatCacheHeader *CacheHdr = NULL; -static inline HeapTuple SearchCatCacheInternal(CatCache *cache, - int nkeys, - Datum v1, Datum v2, - Datum v3, Datum v4); +static HeapTuple SearchCatCacheInternal(CatCache *cache, + int nkeys, + Datum v1, Datum v2, + Datum v3, Datum v4); static pg_noinline HeapTuple SearchCatCacheMiss(CatCache *cache, int nkeys, @@ -75,8 +75,9 @@ static pg_noinline HeapTuple SearchCatCacheMiss(CatCache *cache, Datum v1, Datum v2, Datum v3, Datum v4); -static uint32 CatalogCacheComputeHashValue(CatCache *cache, int nkeys, - Datum v1, Datum v2, Datum v3, Datum v4); +static inline uint32 CatalogCacheComputeHashValue(CatCache *cache, int nkeys, + Datum v1, Datum v2, + Datum v3, Datum v4); static uint32 CatalogCacheComputeTupleHashValue(CatCache *cache, int nkeys, HeapTuple tuple); static inline bool CatalogCacheCompareTuple(const CatCache *cache, int nkeys, @@ -266,7 +267,7 @@ GetCCHashEqFuncs(Oid keytype, CCHashFN *hashfunc, RegProcedure *eqfunc, CCFastEq * * Compute the hash value associated with a given set of lookup keys */ -static uint32 +static inline uint32 CatalogCacheComputeHashValue(CatCache *cache, int nkeys, Datum v1, Datum v2, Datum v3, Datum v4) { @@ -1194,7 +1195,7 @@ SearchCatCache4(CatCache *cache, /* * Work-horse for SearchCatCache/SearchCatCacheN. */ -static inline HeapTuple +static HeapTuple SearchCatCacheInternal(CatCache *cache, int nkeys, Datum v1, -- 2.16.3 ----Next_Part(Mon_Jul_01_16_02_59_2019_106)-- Content-Type: Text/X-Patch; charset=us-ascii Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="v18-0002-Benchmark-extension-and-required-core-change.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v7 1/2] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --3MwIy2ne0vdjdPXF Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0002-allow-syncfs-in-frontend-utilities.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v5 1/2] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 9c8ec779f9..d375dcb795 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --X1bOJ3K7DJ5YkBrT Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-allow-syncfs-in-frontend-utilities.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v6 1/2] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --bp/iNruPH9dso1Pn Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0002-allow-syncfs-in-frontend-utilities.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v7 1/2] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --3MwIy2ne0vdjdPXF Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0002-allow-syncfs-in-frontend-utilities.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v8 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --oyUTqETQ0mS9luUI Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --2fHTh5uZTiUOsy+g Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --2fHTh5uZTiUOsy+g Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --2fHTh5uZTiUOsy+g Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --2fHTh5uZTiUOsy+g Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH 1/1] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --cNdxnHkX5QqsyA0e-- ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v5 1/2] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 9c8ec779f9..d375dcb795 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --X1bOJ3K7DJ5YkBrT Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-allow-syncfs-in-frontend-utilities.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v8 1/4] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/postmaster/postmaster.c | 1 + src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index d7bfb28ff3..54e9bfb8c4 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -94,6 +94,7 @@ #include "access/xlogrecovery.h" #include "catalog/pg_control.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/ip.h" #include "common/pg_prng.h" #include "common/string.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --oyUTqETQ0mS9luUI Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0002-make-common-enum-for-sync-methods.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH 1/1] move PG_TEMP_FILE* macros to file_utils.h @ 2023-08-22 02:05 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Nathan Bossart @ 2023-08-22 02:05 UTC (permalink / raw) --- src/backend/backup/basebackup.c | 2 +- src/backend/storage/file/fileset.c | 1 + src/bin/pg_checksums/pg_checksums.c | 10 ---------- src/bin/pg_rewind/filemap.c | 2 +- src/include/common/file_utils.h | 4 ++++ src/include/storage/fd.h | 4 ---- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 45be21131c..5d66014499 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -25,6 +25,7 @@ #include "commands/defrem.h" #include "common/compression.h" #include "common/file_perm.h" +#include "common/file_utils.h" #include "lib/stringinfo.h" #include "miscadmin.h" #include "nodes/pg_list.h" @@ -37,7 +38,6 @@ #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/dsm_impl.h" -#include "storage/fd.h" #include "storage/ipc.h" #include "storage/reinit.h" #include "utils/builtins.h" diff --git a/src/backend/storage/file/fileset.c b/src/backend/storage/file/fileset.c index e9951b0269..84cae32548 100644 --- a/src/backend/storage/file/fileset.c +++ b/src/backend/storage/file/fileset.c @@ -25,6 +25,7 @@ #include "catalog/pg_tablespace.h" #include "commands/tablespace.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "miscadmin.h" #include "storage/ipc.h" diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 19eb67e485..9011a19b4e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -52,16 +52,6 @@ typedef enum PG_MODE_ENABLE } PgChecksumMode; -/* - * Filename components. - * - * XXX: fd.h is not declared here as frontend side code is not able to - * interact with the backend-side definitions for the various fsync - * wrappers. - */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - static PgChecksumMode mode = PG_MODE_CHECK; static const char *progname; diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index bd5c598e20..58280d9abc 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,12 +27,12 @@ #include <unistd.h> #include "catalog/pg_tablespace_d.h" +#include "common/file_utils.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" #include "filemap.h" #include "pg_rewind.h" -#include "storage/fd.h" /* * Define a hash table which we can use to store information about the files diff --git a/src/include/common/file_utils.h b/src/include/common/file_utils.h index b7efa1226d..dd1532bcb0 100644 --- a/src/include/common/file_utils.h +++ b/src/include/common/file_utils.h @@ -46,4 +46,8 @@ extern ssize_t pg_pwritev_with_retry(int fd, extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset); +/* Filename components */ +#define PG_TEMP_FILES_DIR "pgsql_tmp" +#define PG_TEMP_FILE_PREFIX "pgsql_tmp" + #endif /* FILE_UTILS_H */ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc..aea30c0622 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -195,8 +195,4 @@ extern int durable_unlink(const char *fname, int elevel); extern void SyncDataDirectory(void); extern int data_sync_elevel(int elevel); -/* Filename components */ -#define PG_TEMP_FILES_DIR "pgsql_tmp" -#define PG_TEMP_FILE_PREFIX "pgsql_tmp" - #endif /* FD_H */ -- 2.25.1 --cNdxnHkX5QqsyA0e-- ^ permalink raw reply [nested|flat] 20+ messages in thread
* Re: FileFallocate misbehaving on XFS @ 2024-12-12 21:38 Andres Freund <[email protected]> 0 siblings, 1 reply; 20+ messages in thread From: Andres Freund @ 2024-12-12 21:38 UTC (permalink / raw) To: Michael Harris <[email protected]>; +Cc: Jakub Wartak <[email protected]>; Tomas Vondra <[email protected]>; pgsql-hackers Hi, On 2024-12-12 14:14:20 +1100, Michael Harris wrote: > On Thu, 12 Dec 2024 at 10:50, Andres Freund <[email protected]> wrote: > > Just to make sure - you're absolutely certain that you actually have space at > > the time of the errors? > > As sure as I can be. The RHEL8 system that I took prints from > yesterday has > 1.5TB free. I can't see it varying by that much. That does seem unlikely, but it'd probably still be worth monitoring by how much it varies. > It does look as though the system needs to be quite full to provoke > this problem. The systems I have looked at so far have >90% full > filesystems. > > Another interesting snippet: the application has a number of ETL > workers going at once. The actual number varies depending on a number > of factors but might be somewhere from 10 - 150. Each worker will have > a single postgres backend that they are feeding data to. Are they all inserting into distinct tables/partitions or into shared tables? > At the time of the error, it is not the case that all ETL workers > strike it at once - it looks like a lot of the time only a single > worker is affected, or at most a handful of workers. I can't see for > sure what the other workers were doing at the time, but I would expect > they were all importing data as well. When you say that they're not "all striking it at once", do you mean that some of them aren't interacting with the database at the time, or that they're not erroring out? > > If I were to provide you with a patch that showed the amount of free disk > > space at the time of an error, the size of the relation etc, could you > > reproduce the issue with it applied? Or is that unrealistic? > > I have not been able to reproduce it on demand, and so far it has only > happened in production systems. > > As long as the patch doesn't degrade normal performance it should be > possible to deploy it to one of the systems that is regularly > reporting the error, although it might take a while to get approval to > do that. Cool. The patch only has an effect in the branches reporting out-of-space errors, so there's no overhead during normal operation. And the additional detail doesn't have much overhead in the error case either. I attached separate patches for 16, 17 and master, as there's some minor conflicts between the version. Greetings, Andres Freund Attachments: [text/x-diff] 16-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch (5.7K, ../../vgh2m75nh6s53diujmooue2y6eon3jdriildwqnnoa4okmizmx@3pcg7pa5qubx/2-16-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch) download | inline diff: From c8ecdff54fcdbd2cf89ca7888f641db369f207ce Mon Sep 17 00:00:00 2001 From: Andres Freund <[email protected]> Date: Thu, 12 Dec 2024 12:57:12 -0500 Subject: [PATCH] md: Report more detail when encountering ENOSPC during extension Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- meson.build | 1 + configure.ac | 1 + src/include/pg_config.h.in | 3 ++ src/backend/storage/smgr/md.c | 63 +++++++++++++++++++++++++++++++---- configure | 2 +- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/meson.build b/meson.build index 4e59feb91da..e644db41ef9 100644 --- a/meson.build +++ b/meson.build @@ -2269,6 +2269,7 @@ header_checks = [ 'sys/procctl.h', 'sys/signalfd.h', 'sys/ucred.h', + 'sys/vfs.h', 'termios.h', 'ucred.h', ] diff --git a/configure.ac b/configure.ac index 23add80d8fd..0984949a3b9 100644 --- a/configure.ac +++ b/configure.ac @@ -1512,6 +1512,7 @@ AC_CHECK_HEADERS(m4_normalize([ sys/procctl.h sys/signalfd.h sys/ucred.h + sys/vfs.h termios.h ucred.h ])) diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index ce3063b2b22..626de538821 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -478,6 +478,9 @@ /* Define to 1 if you have the <sys/ucred.h> header file. */ #undef HAVE_SYS_UCRED_H +/* Define to 1 if you have the <sys/vfs.h> header file. */ +#undef HAVE_SYS_VFS_H + /* Define to 1 if you have the <termios.h> header file. */ #undef HAVE_TERMIOS_H diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index fdecbad1709..67c42a69c11 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -24,6 +24,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/file.h> +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif #include "access/xlog.h" #include "access/xlogutils.h" @@ -449,6 +452,37 @@ mdunlinkfork(RelFileLocatorBackend rlocator, ForkNumber forknum, bool isRedo) pfree(path); } +static void +report_disk_space(const char *reason, const char *path) +{ + /* + * I'm sure there's a way to do this on other OSs too, but for the + * debugging here this should be sufficient. + */ +#ifdef HAVE_SYS_VFS_H + int saved_errno = errno; + struct statfs sf; + int ret; + + ret = statfs(path, &sf); + + if (ret != 0) + elog(WARNING, "%s: statfs failed: %m", reason); + else + elog(LOG, "%s: free space for filesystem containing \"%s\" " + "f_blocks: %llu, f_bfree: %llu, f_bavail: %llu " + "f_files: %llu, f_ffree: %llu", + reason, path, + (long long unsigned) sf.f_blocks, + (long long unsigned) sf.f_bfree, + (long long unsigned) sf.f_bavail, + (long long unsigned) sf.f_files, + (long long unsigned) sf.f_ffree); + + errno = saved_errno; +#endif +} + /* * mdextend() -- Add a block to the specified relation. * @@ -496,11 +530,16 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_EXTEND)) != BLCKSZ) { + if (errno == ENOSPC) + report_disk_space("mdextend failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + if (nbytes < 0) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" from %u to %u blocks: %m", + FilePathName(v->mdfd_vfd), + blocknum, blocknum + 1), errhint("Check free disk space."))); /* short write: complain appropriately */ ereport(ERROR, @@ -586,10 +625,15 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, WAIT_EVENT_DATA_FILE_EXTEND); if (ret != 0) { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileFallocate failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\" with FileFallocate(): %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileFallocate(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); } } @@ -608,11 +652,18 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, seekpos, (off_t) BLCKSZ * numblocks, WAIT_EVENT_DATA_FILE_EXTEND); if (ret < 0) + { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileZero failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileZero(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); + } } if (!skipFsync && !SmgrIsTemp(reln)) diff --git a/configure b/configure index 8c2ab3a1973..f62f4f6d3ab 100755 --- a/configure +++ b/configure @@ -13768,7 +13768,7 @@ fi ## Header files ## -for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h langinfo.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h termios.h ucred.h +for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h langinfo.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h sys/vfs.h termios.h ucred.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -- 2.45.2.746.g06e570c0df.dirty [text/x-diff] 17-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch (5.7K, ../../vgh2m75nh6s53diujmooue2y6eon3jdriildwqnnoa4okmizmx@3pcg7pa5qubx/3-17-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch) download | inline diff: From e7119200b69e0c96f85cede510889be9548b0d73 Mon Sep 17 00:00:00 2001 From: Andres Freund <[email protected]> Date: Thu, 12 Dec 2024 12:57:12 -0500 Subject: [PATCH] md: Report more detail when encountering ENOSPC during extension Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- meson.build | 1 + configure.ac | 1 + src/include/pg_config.h.in | 3 ++ src/backend/storage/smgr/md.c | 63 +++++++++++++++++++++++++++++++---- configure | 2 +- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/meson.build b/meson.build index 005dc9f3532..a0113e84aef 100644 --- a/meson.build +++ b/meson.build @@ -2415,6 +2415,7 @@ header_checks = [ 'sys/procctl.h', 'sys/signalfd.h', 'sys/ucred.h', + 'sys/vfs.h', 'termios.h', 'ucred.h', ] diff --git a/configure.ac b/configure.ac index 3c76c9ebc87..074572aabf5 100644 --- a/configure.ac +++ b/configure.ac @@ -1478,6 +1478,7 @@ AC_CHECK_HEADERS(m4_normalize([ sys/procctl.h sys/signalfd.h sys/ucred.h + sys/vfs.h termios.h ucred.h ])) diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 2397d90b465..7b53c994699 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -462,6 +462,9 @@ /* Define to 1 if you have the <sys/ucred.h> header file. */ #undef HAVE_SYS_UCRED_H +/* Define to 1 if you have the <sys/vfs.h> header file. */ +#undef HAVE_SYS_VFS_H + /* Define to 1 if you have the <termios.h> header file. */ #undef HAVE_TERMIOS_H diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 6796756358f..8c49312db0d 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -24,6 +24,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/file.h> +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif #include "access/xlogutils.h" #include "commands/tablespace.h" @@ -447,6 +450,37 @@ mdunlinkfork(RelFileLocatorBackend rlocator, ForkNumber forknum, bool isRedo) pfree(path); } +static void +report_disk_space(const char *reason, const char *path) +{ + /* + * I'm sure there's a way to do this on other OSs too, but for the + * debugging here this should be sufficient. + */ +#ifdef HAVE_SYS_VFS_H + int saved_errno = errno; + struct statfs sf; + int ret; + + ret = statfs(path, &sf); + + if (ret != 0) + elog(WARNING, "%s: statfs failed: %m", reason); + else + elog(LOG, "%s: free space for filesystem containing \"%s\" " + "f_blocks: %llu, f_bfree: %llu, f_bavail: %llu " + "f_files: %llu, f_ffree: %llu", + reason, path, + (long long unsigned) sf.f_blocks, + (long long unsigned) sf.f_bfree, + (long long unsigned) sf.f_bavail, + (long long unsigned) sf.f_files, + (long long unsigned) sf.f_ffree); + + errno = saved_errno; +#endif +} + /* * mdextend() -- Add a block to the specified relation. * @@ -494,11 +528,16 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_EXTEND)) != BLCKSZ) { + if (errno == ENOSPC) + report_disk_space("mdextend failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + if (nbytes < 0) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" from %u to %u blocks: %m", + FilePathName(v->mdfd_vfd), + blocknum, blocknum + 1), errhint("Check free disk space."))); /* short write: complain appropriately */ ereport(ERROR, @@ -584,10 +623,15 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, WAIT_EVENT_DATA_FILE_EXTEND); if (ret != 0) { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileFallocate failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\" with FileFallocate(): %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileFallocate(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); } } @@ -606,11 +650,18 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, seekpos, (off_t) BLCKSZ * numblocks, WAIT_EVENT_DATA_FILE_EXTEND); if (ret < 0) + { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileZero failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileZero(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); + } } if (!skipFsync && !SmgrIsTemp(reln)) diff --git a/configure b/configure index 97996b7f6b7..5efd85bb17a 100755 --- a/configure +++ b/configure @@ -13349,7 +13349,7 @@ fi ## Header files ## -for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h langinfo.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h termios.h ucred.h +for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h langinfo.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h sys/vfs.h termios.h ucred.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -- 2.45.2.746.g06e570c0df.dirty [text/x-diff] HEAD-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch (5.7K, ../../vgh2m75nh6s53diujmooue2y6eon3jdriildwqnnoa4okmizmx@3pcg7pa5qubx/4-HEAD-0001-md-Report-more-detail-when-encountering-ENOSPC-durin.patch) download | inline diff: From 66a18a4565ec4cca4a8ce216aae9322a8c68731c Mon Sep 17 00:00:00 2001 From: Andres Freund <[email protected]> Date: Thu, 12 Dec 2024 12:57:12 -0500 Subject: [PATCH] md: Report more detail when encountering ENOSPC during extension Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- meson.build | 1 + configure.ac | 1 + src/include/pg_config.h.in | 3 ++ src/backend/storage/smgr/md.c | 63 +++++++++++++++++++++++++++++++---- configure | 2 +- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/meson.build b/meson.build index e5ce437a5c7..05f622ccd79 100644 --- a/meson.build +++ b/meson.build @@ -2389,6 +2389,7 @@ header_checks = [ 'sys/procctl.h', 'sys/signalfd.h', 'sys/ucred.h', + 'sys/vfs.h', 'termios.h', 'ucred.h', 'xlocale.h', diff --git a/configure.ac b/configure.ac index 247ae97fa4c..d68774f9c89 100644 --- a/configure.ac +++ b/configure.ac @@ -1446,6 +1446,7 @@ AC_CHECK_HEADERS(m4_normalize([ sys/procctl.h sys/signalfd.h sys/ucred.h + sys/vfs.h termios.h ucred.h xlocale.h diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 07b2f798abd..c5e083f8793 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -439,6 +439,9 @@ /* Define to 1 if you have the <sys/ucred.h> header file. */ #undef HAVE_SYS_UCRED_H +/* Define to 1 if you have the <sys/vfs.h> header file. */ +#undef HAVE_SYS_VFS_H + /* Define to 1 if you have the <termios.h> header file. */ #undef HAVE_TERMIOS_H diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index cc8a80ee961..eac080f1a43 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -24,6 +24,9 @@ #include <unistd.h> #include <fcntl.h> #include <sys/file.h> +#ifdef HAVE_SYS_VFS_H +#include <sys/vfs.h> +#endif #include "access/xlogutils.h" #include "commands/tablespace.h" @@ -447,6 +450,37 @@ mdunlinkfork(RelFileLocatorBackend rlocator, ForkNumber forknum, bool isRedo) pfree(path); } +static void +report_disk_space(const char *reason, const char *path) +{ + /* + * I'm sure there's a way to do this on other OSs too, but for the + * debugging here this should be sufficient. + */ +#ifdef HAVE_SYS_VFS_H + int saved_errno = errno; + struct statfs sf; + int ret; + + ret = statfs(path, &sf); + + if (ret != 0) + elog(WARNING, "%s: statfs failed: %m", reason); + else + elog(LOG, "%s: free space for filesystem containing \"%s\" " + "f_blocks: %llu, f_bfree: %llu, f_bavail: %llu " + "f_files: %llu, f_ffree: %llu", + reason, path, + (long long unsigned) sf.f_blocks, + (long long unsigned) sf.f_bfree, + (long long unsigned) sf.f_bavail, + (long long unsigned) sf.f_files, + (long long unsigned) sf.f_ffree); + + errno = saved_errno; +#endif +} + /* * mdextend() -- Add a block to the specified relation. * @@ -494,11 +528,16 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ, seekpos, WAIT_EVENT_DATA_FILE_EXTEND)) != BLCKSZ) { + if (errno == ENOSPC) + report_disk_space("mdextend failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + if (nbytes < 0) ereport(ERROR, (errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" from %u to %u blocks: %m", + FilePathName(v->mdfd_vfd), + blocknum, blocknum + 1), errhint("Check free disk space."))); /* short write: complain appropriately */ ereport(ERROR, @@ -584,10 +623,15 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, WAIT_EVENT_DATA_FILE_EXTEND); if (ret != 0) { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileFallocate failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\" with FileFallocate(): %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileFallocate(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); } } @@ -606,11 +650,18 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, seekpos, (off_t) BLCKSZ * numblocks, WAIT_EVENT_DATA_FILE_EXTEND); if (ret < 0) + { + if (errno == ENOSPC) + report_disk_space("mdzeroextend FileZero failing with ENOSPC", + FilePathName(v->mdfd_vfd)); + ereport(ERROR, errcode_for_file_access(), - errmsg("could not extend file \"%s\": %m", - FilePathName(v->mdfd_vfd)), + errmsg("could not extend file \"%s\" by %u blocks, from %u to %u, using FileZero(): %m", + FilePathName(v->mdfd_vfd), + numblocks, segstartblock, segstartblock+numblocks), errhint("Check free disk space.")); + } } if (!skipFsync && !SmgrIsTemp(reln)) diff --git a/configure b/configure index 518c33b73a9..191cbca0844 100755 --- a/configure +++ b/configure @@ -13227,7 +13227,7 @@ fi ## Header files ## -for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h termios.h ucred.h xlocale.h +for ac_header in atomic.h copyfile.h execinfo.h getopt.h ifaddrs.h mbarrier.h sys/epoll.h sys/event.h sys/personality.h sys/prctl.h sys/procctl.h sys/signalfd.h sys/ucred.h sys/vfs.h termios.h ucred.h xlocale.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -- 2.45.2.746.g06e570c0df.dirty ^ permalink raw reply [nested|flat] 20+ messages in thread
* Re: FileFallocate misbehaving on XFS @ 2024-12-12 22:13 Michael Harris <[email protected]> parent: Andres Freund <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Michael Harris @ 2024-12-12 22:13 UTC (permalink / raw) To: Andres Freund <[email protected]>; +Cc: Jakub Wartak <[email protected]>; Tomas Vondra <[email protected]>; pgsql-hackers Hi Andres On Fri, 13 Dec 2024 at 08:38, Andres Freund <[email protected]> wrote: > > Another interesting snippet: the application has a number of ETL > > workers going at once. The actual number varies depending on a number > > of factors but might be somewhere from 10 - 150. Each worker will have > > a single postgres backend that they are feeding data to. > > Are they all inserting into distinct tables/partitions or into shared tables? The set of tables they are writing into is the same, but we do take some effort to randomize the order of the tables that we each worker is writing into so as to reduce contention. Even so it is quite likely that multiple processes will be writing into a table at a time. Also worth noting that I have only seen this error triggered by COPY statements (other than the upgrade case). There are some other cases in our code that use INSERT but so far I have not seen that end in an out of space error. > When you say that they're not "all striking it at once", do you mean that some > of them aren't interacting with the database at the time, or that they're not > erroring out? Sorry, I meant erroring out. Thanks for the patch! Cheers Mike ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 05/10] bufmgr: Make definitions related to buffer descriptor easier to modify @ 2026-01-07 22:21 Andres Freund <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Andres Freund @ 2026-01-07 22:21 UTC (permalink / raw) This is in preparation to widening the buffer state to 64 bits, which in turn is preparation for implementing content locks in bufmgr. This commit aims to make the subsequent commits a bit easier to review, by separating out reformatting etc from the actual changes. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/include/storage/buf_internals.h | 65 +++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index fa43cf4458d..2f607ea2ac5 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -32,6 +32,7 @@ /* * Buffer state is a single 32-bit variable where following data is combined. * + * State of the buffer itself (in order): * - 18 bits refcount * - 4 bits usage count * - 10 bits of flags @@ -48,16 +49,30 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, "parts of buffer state space need to equal 32"); +/* refcount related definitions */ #define BUF_REFCOUNT_ONE 1 -#define BUF_REFCOUNT_MASK ((1U << BUF_REFCOUNT_BITS) - 1) -#define BUF_USAGECOUNT_MASK (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_REFCOUNT_BITS)) -#define BUF_USAGECOUNT_ONE (1U << BUF_REFCOUNT_BITS) -#define BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS -#define BUF_FLAG_MASK (((1U << BUF_FLAG_BITS) - 1) << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)) +#define BUF_REFCOUNT_MASK \ + ((1U << BUF_REFCOUNT_BITS) - 1) + +/* usage count related definitions */ +#define BUF_USAGECOUNT_SHIFT \ + BUF_REFCOUNT_BITS +#define BUF_USAGECOUNT_MASK \ + (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT)) +#define BUF_USAGECOUNT_ONE \ + (1U << BUF_REFCOUNT_BITS) + +/* flags related definitions */ +#define BUF_FLAG_SHIFT \ + (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) +#define BUF_FLAG_MASK \ + (((1U << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT) /* Get refcount and usagecount from buffer state */ -#define BUF_STATE_GET_REFCOUNT(state) ((state) & BUF_REFCOUNT_MASK) -#define BUF_STATE_GET_USAGECOUNT(state) (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) +#define BUF_STATE_GET_REFCOUNT(state) \ + ((state) & BUF_REFCOUNT_MASK) +#define BUF_STATE_GET_USAGECOUNT(state) \ + (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) /* * Flags for buffer descriptors @@ -65,17 +80,31 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, * Note: BM_TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ -#define BM_LOCKED (1U << 22) /* buffer header is locked */ -#define BM_DIRTY (1U << 23) /* data needs writing */ -#define BM_VALID (1U << 24) /* data is valid */ -#define BM_TAG_VALID (1U << 25) /* tag is assigned */ -#define BM_IO_IN_PROGRESS (1U << 26) /* read or write in progress */ -#define BM_IO_ERROR (1U << 27) /* previous I/O failed */ -#define BM_JUST_DIRTIED (1U << 28) /* dirtied since write started */ -#define BM_PIN_COUNT_WAITER (1U << 29) /* have waiter for sole pin */ -#define BM_CHECKPOINT_NEEDED (1U << 30) /* must write for checkpoint */ -#define BM_PERMANENT (1U << 31) /* permanent buffer (not unlogged, - * or init fork) */ + +#define BUF_DEFINE_FLAG(flagno) \ + (1U << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + (flagno))) + +/* buffer header is locked */ +#define BM_LOCKED BUF_DEFINE_FLAG( 0) +/* data needs writing */ +#define BM_DIRTY BUF_DEFINE_FLAG( 1) +/* data is valid */ +#define BM_VALID BUF_DEFINE_FLAG( 2) +/* tag is assigned */ +#define BM_TAG_VALID BUF_DEFINE_FLAG( 3) +/* read or write in progress */ +#define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4) +/* previous I/O failed */ +#define BM_IO_ERROR BUF_DEFINE_FLAG( 5) +/* dirtied since write started */ +#define BM_JUST_DIRTIED BUF_DEFINE_FLAG( 6) +/* have waiter for sole pin */ +#define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7) +/* must write for checkpoint */ +#define BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8) +/* permanent buffer (not unlogged, or init fork) */ +#define BM_PERMANENT BUF_DEFINE_FLAG( 9) + /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A -- 2.48.1.76.g4e746b1a31.dirty --rkiyqpij3ajqn7ww Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0006-bufmgr-Change-BufferDesc.state-to-be-a-64bit-atom.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v10 2/8] bufmgr: Make definitions related to buffer descriptor easier to modify @ 2026-01-07 22:21 Andres Freund <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Andres Freund @ 2026-01-07 22:21 UTC (permalink / raw) This is in preparation to widening the buffer state to 64 bits, which in turn is preparation for implementing content locks in bufmgr. This commit aims to make the subsequent commits a bit easier to review, by separating out reformatting etc from the actual changes. Discussion: https://postgr.es/m/4csodkvvfbfloxxjlkgsnl2lgfv2mtzdl7phqzd4jxjadxm4o5@usw7feyb5bzf --- src/include/storage/buf_internals.h | 65 +++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index fa43cf4458d..2f607ea2ac5 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -32,6 +32,7 @@ /* * Buffer state is a single 32-bit variable where following data is combined. * + * State of the buffer itself (in order): * - 18 bits refcount * - 4 bits usage count * - 10 bits of flags @@ -48,16 +49,30 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, "parts of buffer state space need to equal 32"); +/* refcount related definitions */ #define BUF_REFCOUNT_ONE 1 -#define BUF_REFCOUNT_MASK ((1U << BUF_REFCOUNT_BITS) - 1) -#define BUF_USAGECOUNT_MASK (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_REFCOUNT_BITS)) -#define BUF_USAGECOUNT_ONE (1U << BUF_REFCOUNT_BITS) -#define BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS -#define BUF_FLAG_MASK (((1U << BUF_FLAG_BITS) - 1) << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)) +#define BUF_REFCOUNT_MASK \ + ((1U << BUF_REFCOUNT_BITS) - 1) + +/* usage count related definitions */ +#define BUF_USAGECOUNT_SHIFT \ + BUF_REFCOUNT_BITS +#define BUF_USAGECOUNT_MASK \ + (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT)) +#define BUF_USAGECOUNT_ONE \ + (1U << BUF_REFCOUNT_BITS) + +/* flags related definitions */ +#define BUF_FLAG_SHIFT \ + (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) +#define BUF_FLAG_MASK \ + (((1U << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT) /* Get refcount and usagecount from buffer state */ -#define BUF_STATE_GET_REFCOUNT(state) ((state) & BUF_REFCOUNT_MASK) -#define BUF_STATE_GET_USAGECOUNT(state) (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) +#define BUF_STATE_GET_REFCOUNT(state) \ + ((state) & BUF_REFCOUNT_MASK) +#define BUF_STATE_GET_USAGECOUNT(state) \ + (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) /* * Flags for buffer descriptors @@ -65,17 +80,31 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, * Note: BM_TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ -#define BM_LOCKED (1U << 22) /* buffer header is locked */ -#define BM_DIRTY (1U << 23) /* data needs writing */ -#define BM_VALID (1U << 24) /* data is valid */ -#define BM_TAG_VALID (1U << 25) /* tag is assigned */ -#define BM_IO_IN_PROGRESS (1U << 26) /* read or write in progress */ -#define BM_IO_ERROR (1U << 27) /* previous I/O failed */ -#define BM_JUST_DIRTIED (1U << 28) /* dirtied since write started */ -#define BM_PIN_COUNT_WAITER (1U << 29) /* have waiter for sole pin */ -#define BM_CHECKPOINT_NEEDED (1U << 30) /* must write for checkpoint */ -#define BM_PERMANENT (1U << 31) /* permanent buffer (not unlogged, - * or init fork) */ + +#define BUF_DEFINE_FLAG(flagno) \ + (1U << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + (flagno))) + +/* buffer header is locked */ +#define BM_LOCKED BUF_DEFINE_FLAG( 0) +/* data needs writing */ +#define BM_DIRTY BUF_DEFINE_FLAG( 1) +/* data is valid */ +#define BM_VALID BUF_DEFINE_FLAG( 2) +/* tag is assigned */ +#define BM_TAG_VALID BUF_DEFINE_FLAG( 3) +/* read or write in progress */ +#define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4) +/* previous I/O failed */ +#define BM_IO_ERROR BUF_DEFINE_FLAG( 5) +/* dirtied since write started */ +#define BM_JUST_DIRTIED BUF_DEFINE_FLAG( 6) +/* have waiter for sole pin */ +#define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7) +/* must write for checkpoint */ +#define BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8) +/* permanent buffer (not unlogged, or init fork) */ +#define BM_PERMANENT BUF_DEFINE_FLAG( 9) + /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A -- 2.48.1.76.g4e746b1a31.dirty --jmawlk4t5yqwiemy Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0003-bufmgr-Change-BufferDesc.state-to-be-a-64-bit-at.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v9 05/10] bufmgr: Make definitions related to buffer descriptor easier to modify @ 2026-01-07 22:21 Andres Freund <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Andres Freund @ 2026-01-07 22:21 UTC (permalink / raw) This is in preparation to widening the buffer state to 64 bits, which in turn is preparation for implementing content locks in bufmgr. This commit aims to make the subsequent commits a bit easier to review, by separating out reformatting etc from the actual changes. Author: Reviewed-by: Discussion: https://postgr.es/m/ Backpatch: --- src/include/storage/buf_internals.h | 65 +++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index fa43cf4458d..2f607ea2ac5 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -32,6 +32,7 @@ /* * Buffer state is a single 32-bit variable where following data is combined. * + * State of the buffer itself (in order): * - 18 bits refcount * - 4 bits usage count * - 10 bits of flags @@ -48,16 +49,30 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, "parts of buffer state space need to equal 32"); +/* refcount related definitions */ #define BUF_REFCOUNT_ONE 1 -#define BUF_REFCOUNT_MASK ((1U << BUF_REFCOUNT_BITS) - 1) -#define BUF_USAGECOUNT_MASK (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_REFCOUNT_BITS)) -#define BUF_USAGECOUNT_ONE (1U << BUF_REFCOUNT_BITS) -#define BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS -#define BUF_FLAG_MASK (((1U << BUF_FLAG_BITS) - 1) << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)) +#define BUF_REFCOUNT_MASK \ + ((1U << BUF_REFCOUNT_BITS) - 1) + +/* usage count related definitions */ +#define BUF_USAGECOUNT_SHIFT \ + BUF_REFCOUNT_BITS +#define BUF_USAGECOUNT_MASK \ + (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT)) +#define BUF_USAGECOUNT_ONE \ + (1U << BUF_REFCOUNT_BITS) + +/* flags related definitions */ +#define BUF_FLAG_SHIFT \ + (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) +#define BUF_FLAG_MASK \ + (((1U << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT) /* Get refcount and usagecount from buffer state */ -#define BUF_STATE_GET_REFCOUNT(state) ((state) & BUF_REFCOUNT_MASK) -#define BUF_STATE_GET_USAGECOUNT(state) (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) +#define BUF_STATE_GET_REFCOUNT(state) \ + ((state) & BUF_REFCOUNT_MASK) +#define BUF_STATE_GET_USAGECOUNT(state) \ + (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) /* * Flags for buffer descriptors @@ -65,17 +80,31 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, * Note: BM_TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ -#define BM_LOCKED (1U << 22) /* buffer header is locked */ -#define BM_DIRTY (1U << 23) /* data needs writing */ -#define BM_VALID (1U << 24) /* data is valid */ -#define BM_TAG_VALID (1U << 25) /* tag is assigned */ -#define BM_IO_IN_PROGRESS (1U << 26) /* read or write in progress */ -#define BM_IO_ERROR (1U << 27) /* previous I/O failed */ -#define BM_JUST_DIRTIED (1U << 28) /* dirtied since write started */ -#define BM_PIN_COUNT_WAITER (1U << 29) /* have waiter for sole pin */ -#define BM_CHECKPOINT_NEEDED (1U << 30) /* must write for checkpoint */ -#define BM_PERMANENT (1U << 31) /* permanent buffer (not unlogged, - * or init fork) */ + +#define BUF_DEFINE_FLAG(flagno) \ + (1U << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + (flagno))) + +/* buffer header is locked */ +#define BM_LOCKED BUF_DEFINE_FLAG( 0) +/* data needs writing */ +#define BM_DIRTY BUF_DEFINE_FLAG( 1) +/* data is valid */ +#define BM_VALID BUF_DEFINE_FLAG( 2) +/* tag is assigned */ +#define BM_TAG_VALID BUF_DEFINE_FLAG( 3) +/* read or write in progress */ +#define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4) +/* previous I/O failed */ +#define BM_IO_ERROR BUF_DEFINE_FLAG( 5) +/* dirtied since write started */ +#define BM_JUST_DIRTIED BUF_DEFINE_FLAG( 6) +/* have waiter for sole pin */ +#define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7) +/* must write for checkpoint */ +#define BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8) +/* permanent buffer (not unlogged, or init fork) */ +#define BM_PERMANENT BUF_DEFINE_FLAG( 9) + /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A -- 2.48.1.76.g4e746b1a31.dirty --rkiyqpij3ajqn7ww Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0006-bufmgr-Change-BufferDesc.state-to-be-a-64bit-atom.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
* [PATCH v10 2/8] bufmgr: Make definitions related to buffer descriptor easier to modify @ 2026-01-07 22:21 Andres Freund <[email protected]> 0 siblings, 0 replies; 20+ messages in thread From: Andres Freund @ 2026-01-07 22:21 UTC (permalink / raw) This is in preparation to widening the buffer state to 64 bits, which in turn is preparation for implementing content locks in bufmgr. This commit aims to make the subsequent commits a bit easier to review, by separating out reformatting etc from the actual changes. Discussion: https://postgr.es/m/4csodkvvfbfloxxjlkgsnl2lgfv2mtzdl7phqzd4jxjadxm4o5@usw7feyb5bzf --- src/include/storage/buf_internals.h | 65 +++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index fa43cf4458d..2f607ea2ac5 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -32,6 +32,7 @@ /* * Buffer state is a single 32-bit variable where following data is combined. * + * State of the buffer itself (in order): * - 18 bits refcount * - 4 bits usage count * - 10 bits of flags @@ -48,16 +49,30 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, "parts of buffer state space need to equal 32"); +/* refcount related definitions */ #define BUF_REFCOUNT_ONE 1 -#define BUF_REFCOUNT_MASK ((1U << BUF_REFCOUNT_BITS) - 1) -#define BUF_USAGECOUNT_MASK (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_REFCOUNT_BITS)) -#define BUF_USAGECOUNT_ONE (1U << BUF_REFCOUNT_BITS) -#define BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS -#define BUF_FLAG_MASK (((1U << BUF_FLAG_BITS) - 1) << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)) +#define BUF_REFCOUNT_MASK \ + ((1U << BUF_REFCOUNT_BITS) - 1) + +/* usage count related definitions */ +#define BUF_USAGECOUNT_SHIFT \ + BUF_REFCOUNT_BITS +#define BUF_USAGECOUNT_MASK \ + (((1U << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT)) +#define BUF_USAGECOUNT_ONE \ + (1U << BUF_REFCOUNT_BITS) + +/* flags related definitions */ +#define BUF_FLAG_SHIFT \ + (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) +#define BUF_FLAG_MASK \ + (((1U << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT) /* Get refcount and usagecount from buffer state */ -#define BUF_STATE_GET_REFCOUNT(state) ((state) & BUF_REFCOUNT_MASK) -#define BUF_STATE_GET_USAGECOUNT(state) (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) +#define BUF_STATE_GET_REFCOUNT(state) \ + ((state) & BUF_REFCOUNT_MASK) +#define BUF_STATE_GET_USAGECOUNT(state) \ + (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT) /* * Flags for buffer descriptors @@ -65,17 +80,31 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32, * Note: BM_TAG_VALID essentially means that there is a buffer hashtable * entry associated with the buffer's tag. */ -#define BM_LOCKED (1U << 22) /* buffer header is locked */ -#define BM_DIRTY (1U << 23) /* data needs writing */ -#define BM_VALID (1U << 24) /* data is valid */ -#define BM_TAG_VALID (1U << 25) /* tag is assigned */ -#define BM_IO_IN_PROGRESS (1U << 26) /* read or write in progress */ -#define BM_IO_ERROR (1U << 27) /* previous I/O failed */ -#define BM_JUST_DIRTIED (1U << 28) /* dirtied since write started */ -#define BM_PIN_COUNT_WAITER (1U << 29) /* have waiter for sole pin */ -#define BM_CHECKPOINT_NEEDED (1U << 30) /* must write for checkpoint */ -#define BM_PERMANENT (1U << 31) /* permanent buffer (not unlogged, - * or init fork) */ + +#define BUF_DEFINE_FLAG(flagno) \ + (1U << (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + (flagno))) + +/* buffer header is locked */ +#define BM_LOCKED BUF_DEFINE_FLAG( 0) +/* data needs writing */ +#define BM_DIRTY BUF_DEFINE_FLAG( 1) +/* data is valid */ +#define BM_VALID BUF_DEFINE_FLAG( 2) +/* tag is assigned */ +#define BM_TAG_VALID BUF_DEFINE_FLAG( 3) +/* read or write in progress */ +#define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4) +/* previous I/O failed */ +#define BM_IO_ERROR BUF_DEFINE_FLAG( 5) +/* dirtied since write started */ +#define BM_JUST_DIRTIED BUF_DEFINE_FLAG( 6) +/* have waiter for sole pin */ +#define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7) +/* must write for checkpoint */ +#define BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8) +/* permanent buffer (not unlogged, or init fork) */ +#define BM_PERMANENT BUF_DEFINE_FLAG( 9) + /* * The maximum allowed value of usage_count represents a tradeoff between * accuracy and speed of the clock-sweep buffer management algorithm. A -- 2.48.1.76.g4e746b1a31.dirty --jmawlk4t5yqwiemy Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0003-bufmgr-Change-BufferDesc.state-to-be-a-64-bit-at.patch" ^ permalink raw reply [nested|flat] 20+ messages in thread
end of thread, other threads:[~2026-01-07 22:21 UTC | newest] Thread overview: 20+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-06-28 07:29 [PATCH 1/5] Adjust inlining of some functions Kyotaro Horiguchi <[email protected]> 2023-08-22 02:05 [PATCH v8 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v6 1/2] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v8 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH 1/1] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v7 1/2] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v5 1/2] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH 1/1] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v5 1/2] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v9 1/4] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2023-08-22 02:05 [PATCH v7 1/2] move PG_TEMP_FILE* macros to file_utils.h Nathan Bossart <[email protected]> 2024-12-12 21:38 Re: FileFallocate misbehaving on XFS Andres Freund <[email protected]> 2024-12-12 22:13 ` Re: FileFallocate misbehaving on XFS Michael Harris <[email protected]> 2026-01-07 22:21 [PATCH v10 2/8] bufmgr: Make definitions related to buffer descriptor easier to modify Andres Freund <[email protected]> 2026-01-07 22:21 [PATCH v9 05/10] bufmgr: Make definitions related to buffer descriptor easier to modify Andres Freund <[email protected]> 2026-01-07 22:21 [PATCH v10 2/8] bufmgr: Make definitions related to buffer descriptor easier to modify Andres Freund <[email protected]> 2026-01-07 22:21 [PATCH v9 05/10] bufmgr: Make definitions related to buffer descriptor easier to modify Andres Freund <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox