agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 03/20] Support multiple compression algs/levels/opts.. 9+ messages / 2 participants [nested] [flat]
* [PATCH 03/20] Support multiple compression algs/levels/opts.. @ 2020-12-22 06:23 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 9+ messages in thread From: Justin Pryzby @ 2020-12-22 06:23 UTC (permalink / raw) The existing implementation abtracts compressed and noncompressed I/O. This preliminary commit is intended to also allow for multiple compression algorithms. --- src/bin/pg_dump/compress_io.c | 220 +++++++++++--------------- src/bin/pg_dump/compress_io.h | 19 +-- src/bin/pg_dump/pg_backup.h | 23 ++- src/bin/pg_dump/pg_backup_archiver.c | 45 +++--- src/bin/pg_dump/pg_backup_archiver.h | 12 +- src/bin/pg_dump/pg_backup_custom.c | 6 +- src/bin/pg_dump/pg_backup_directory.c | 17 +- src/bin/pg_dump/pg_backup_tar.c | 33 ++-- src/bin/pg_dump/pg_dump.c | 115 +++++++++++--- 9 files changed, 272 insertions(+), 218 deletions(-) diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c index 6a428978d4..db16fd33f2 100644 --- a/src/bin/pg_dump/compress_io.c +++ b/src/bin/pg_dump/compress_io.c @@ -74,12 +74,9 @@ struct CompressorState #endif }; -static void ParseCompressionOption(int compression, CompressionAlgorithm *alg, - int *level); - /* Routines that support zlib compressed data I/O */ #ifdef HAVE_LIBZ -static void InitCompressorZlib(CompressorState *cs, int level); +static void InitCompressorZlib(CompressorState *cs, Compress *compress); static void DeflateCompressorZlib(ArchiveHandle *AH, CompressorState *cs, bool flush); static void ReadDataFromArchiveZlib(ArchiveHandle *AH, ReadFunc readF); @@ -93,58 +90,36 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF); static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs, const char *data, size_t dLen); -/* - * Interprets a numeric 'compression' value. The algorithm implied by the - * value (zlib or none at the moment), is returned in *alg, and the - * zlib compression level in *level. - */ -static void -ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level) -{ - if (compression == Z_DEFAULT_COMPRESSION || - (compression > 0 && compression <= 9)) - *alg = COMPR_ALG_LIBZ; - else if (compression == 0) - *alg = COMPR_ALG_NONE; - else - { - fatal("invalid compression code: %d", compression); - *alg = COMPR_ALG_NONE; /* keep compiler quiet */ - } - - /* The level is just the passed-in value. */ - if (level) - *level = compression; -} - /* Public interface routines */ /* Allocate a new compressor */ CompressorState * -AllocateCompressor(int compression, WriteFunc writeF) +AllocateCompressor(Compress *compression, WriteFunc writeF) { CompressorState *cs; - CompressionAlgorithm alg; - int level; - - ParseCompressionOption(compression, &alg, &level); - -#ifndef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - fatal("not built with zlib support"); -#endif cs = (CompressorState *) pg_malloc0(sizeof(CompressorState)); cs->writeF = writeF; - cs->comprAlg = alg; + cs->comprAlg = compression->alg; /* * Perform compression algorithm specific initialization. */ + Assert (compression->alg != COMPR_ALG_DEFAULT); + switch (compression->alg) + { #ifdef HAVE_LIBZ - if (alg == COMPR_ALG_LIBZ) - InitCompressorZlib(cs, level); + case COMPR_ALG_LIBZ: + InitCompressorZlib(cs, compression); + break; #endif + case COMPR_ALG_NONE: + /* Do nothing */ + break; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } return cs; } @@ -154,21 +129,21 @@ AllocateCompressor(int compression, WriteFunc writeF) * out with ahwrite(). */ void -ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF) +ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF) { - CompressionAlgorithm alg; - - ParseCompressionOption(compression, &alg, NULL); - - if (alg == COMPR_ALG_NONE) - ReadDataFromArchiveNone(AH, readF); - if (alg == COMPR_ALG_LIBZ) + switch (AH->compression.alg) { + case COMPR_ALG_NONE: + ReadDataFromArchiveNone(AH, readF); + break; #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: ReadDataFromArchiveZlib(AH, readF); -#else - fatal("not built with zlib support"); + break; #endif + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -181,16 +156,18 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, { switch (cs->comprAlg) { - case COMPR_ALG_LIBZ: #ifdef HAVE_LIBZ + case COMPR_ALG_LIBZ: WriteDataToArchiveZlib(AH, cs, data, dLen); -#else - fatal("not built with zlib support"); -#endif break; +#endif case COMPR_ALG_NONE: WriteDataToArchiveNone(AH, cs, data, dLen); break; + + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); } } @@ -215,7 +192,7 @@ EndCompressor(ArchiveHandle *AH, CompressorState *cs) */ static void -InitCompressorZlib(CompressorState *cs, int level) +InitCompressorZlib(CompressorState *cs, Compress *compress) { z_streamp zp; @@ -232,7 +209,7 @@ InitCompressorZlib(CompressorState *cs, int level) cs->zlibOut = (char *) pg_malloc(ZLIB_OUT_SIZE + 1); cs->zlibOutSize = ZLIB_OUT_SIZE; - if (deflateInit(zp, level) != Z_OK) + if (deflateInit(zp, compress->level) != Z_OK) fatal("could not initialize compression library: %s", zp->msg); @@ -424,9 +401,7 @@ struct cfp #endif }; -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix); -#endif /* free() without changing errno; useful in several places below */ static void @@ -442,34 +417,31 @@ free_keep_errno(void *p) * Open a file for reading. 'path' is the file to open, and 'mode' should * be either "r" or "rb". * - * If the file at 'path' does not exist, we append the ".gz" suffix (if 'path' - * doesn't already have it) and try again. So if you pass "foo" as 'path', + * If the file at 'path' does not exist, we search with compressed suffix (if 'path' + * doesn't already have one) and try again. So if you pass "foo" as 'path', * this will open either "foo" or "foo.gz". * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_read(const char *path, const char *mode) +cfopen_read(const char *path, const char *mode, Compress *compression) { cfp *fp; -#ifdef HAVE_LIBZ if (hasSuffix(path, ".gz")) - fp = cfopen(path, mode, 1); + fp = cfopen(path, mode, compression); else -#endif { - fp = cfopen(path, mode, 0); -#ifdef HAVE_LIBZ + fp = cfopen(path, mode, compression); if (fp == NULL) { char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); - fp = cfopen(fname, mode, 1); + fname = psprintf("%s%s", path, suffix); + fp = cfopen(fname, mode, compression); free_keep_errno(fname); } -#endif } return fp; } @@ -479,31 +451,26 @@ cfopen_read(const char *path, const char *mode) * be a filemode as accepted by fopen() and gzopen() that indicates writing * ("w", "wb", "a", or "ab"). * - * If 'compression' is non-zero, a gzip compressed stream is opened, and - * 'compression' indicates the compression level used. The ".gz" suffix - * is automatically added to 'path' in that case. + * Use compression if specified. + * The appropriate suffix is automatically added to 'path' in that case. * * On failure, return NULL with an error code in errno. */ cfp * -cfopen_write(const char *path, const char *mode, int compression) +cfopen_write(const char *path, const char *mode, Compress *compression) { cfp *fp; - if (compression == 0) - fp = cfopen(path, mode, 0); + if (compression->alg == COMPR_ALG_NONE) + fp = cfopen(path, mode, compression); else { -#ifdef HAVE_LIBZ char *fname; + const char *suffix = compress_suffix(compression); - fname = psprintf("%s.gz", path); + fname = psprintf("%s%s", path, suffix); fp = cfopen(fname, mode, compression); free_keep_errno(fname); -#else - fatal("not built with zlib support"); - fp = NULL; /* keep compiler quiet */ -#endif } return fp; } @@ -515,20 +482,21 @@ cfopen_write(const char *path, const char *mode, int compression) * On failure, return NULL with an error code in errno. */ cfp * -cfopen(const char *path, const char *mode, int compression) +cfopen(const char *path, const char *mode, Compress *compression) { - cfp *fp = pg_malloc(sizeof(cfp)); + cfp *fp = pg_malloc0(sizeof(cfp)); - if (compression != 0) + switch (compression->alg) { #ifdef HAVE_LIBZ - if (compression != Z_DEFAULT_COMPRESSION) + case COMPR_ALG_LIBZ: + if (compression->level != Z_DEFAULT_COMPRESSION) { /* user has specified a compression level, so tell zlib to use it */ char mode_compression[32]; snprintf(mode_compression, sizeof(mode_compression), "%s%d", - mode, compression); + mode, compression->level); fp->compressedfp = gzopen(path, mode_compression); } else @@ -537,30 +505,27 @@ cfopen(const char *path, const char *mode, int compression) fp->compressedfp = gzopen(path, mode); } - fp->uncompressedfp = NULL; if (fp->compressedfp == NULL) { free_keep_errno(fp); fp = NULL; } -#else - fatal("not built with zlib support"); -#endif - } - else - { -#ifdef HAVE_LIBZ - fp->compressedfp = NULL; + return fp; #endif + + case COMPR_ALG_NONE: fp->uncompressedfp = fopen(path, mode); if (fp->uncompressedfp == NULL) { free_keep_errno(fp); fp = NULL; } - } + return fp; - return fp; + default: + /* Should not happen */ + fatal("requested compression not available in this installation"); + } } @@ -584,14 +549,13 @@ cfread(void *ptr, int size, cfp *fp) fatal("could not read from input file: %s", errnum == Z_ERRNO ? strerror(errno) : errmsg); } + return ret; } - else #endif - { - ret = fread(ptr, 1, size, fp->uncompressedfp); - if (ret != size && !feof(fp->uncompressedfp)) - READ_ERROR_EXIT(fp->uncompressedfp); - } + + ret = fread(ptr, 1, size, fp->uncompressedfp); + if (ret != size && !feof(fp->uncompressedfp)) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -601,9 +565,8 @@ cfwrite(const void *ptr, int size, cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzwrite(fp->compressedfp, ptr, size); - else #endif - return fwrite(ptr, 1, size, fp->uncompressedfp); + return fwrite(ptr, 1, size, fp->uncompressedfp); } int @@ -622,15 +585,12 @@ cfgetc(cfp *fp) else fatal("could not read from input file: end of file"); } + return ret; } - else #endif - { - ret = fgetc(fp->uncompressedfp); - if (ret == EOF) - READ_ERROR_EXIT(fp->uncompressedfp); - } - + ret = fgetc(fp->uncompressedfp); + if (ret == EOF) + READ_ERROR_EXIT(fp->uncompressedfp); return ret; } @@ -640,9 +600,8 @@ cfgets(cfp *fp, char *buf, int len) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzgets(fp->compressedfp, buf, len); - else #endif - return fgets(buf, len, fp->uncompressedfp); + return fgets(buf, len, fp->uncompressedfp); } int @@ -660,15 +619,13 @@ cfclose(cfp *fp) { result = gzclose(fp->compressedfp); fp->compressedfp = NULL; + return result; } - else #endif - { - result = fclose(fp->uncompressedfp); - fp->uncompressedfp = NULL; - } - free_keep_errno(fp); + result = fclose(fp->uncompressedfp); + fp->uncompressedfp = NULL; + free_keep_errno(fp); return result; } @@ -678,9 +635,9 @@ cfeof(cfp *fp) #ifdef HAVE_LIBZ if (fp->compressedfp) return gzeof(fp->compressedfp); - else #endif - return feof(fp->uncompressedfp); + + return feof(fp->uncompressedfp); } const char * @@ -699,7 +656,6 @@ get_cfp_error(cfp *fp) return strerror(errno); } -#ifdef HAVE_LIBZ static int hasSuffix(const char *filename, const char *suffix) { @@ -714,4 +670,18 @@ hasSuffix(const char *filename, const char *suffix) suffixlen) == 0; } -#endif +/* + * Return a string for the given AH's compression. + * The string is statically allocated. + */ +const char * +compress_suffix(Compress *compression) +{ + switch (compression->alg) + { + case COMPR_ALG_LIBZ: + return ".gz"; + default: + return ""; + } +} diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h index d2e6e1b854..2c073676eb 100644 --- a/src/bin/pg_dump/compress_io.h +++ b/src/bin/pg_dump/compress_io.h @@ -21,12 +21,6 @@ #define ZLIB_OUT_SIZE 4096 #define ZLIB_IN_SIZE 4096 -typedef enum -{ - COMPR_ALG_NONE, - COMPR_ALG_LIBZ -} CompressionAlgorithm; - /* Prototype for callback function to WriteDataToArchive() */ typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len); @@ -46,8 +40,8 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen); /* struct definition appears in compress_io.c */ typedef struct CompressorState CompressorState; -extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF); -extern void ReadDataFromArchive(ArchiveHandle *AH, int compression, +extern CompressorState *AllocateCompressor(Compress *compression, WriteFunc writeF); +extern void ReadDataFromArchive(ArchiveHandle *AH, ReadFunc readF); extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs, const void *data, size_t dLen); @@ -56,9 +50,9 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs); typedef struct cfp cfp; -extern cfp *cfopen(const char *path, const char *mode, int compression); -extern cfp *cfopen_read(const char *path, const char *mode); -extern cfp *cfopen_write(const char *path, const char *mode, int compression); +extern cfp *cfopen(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_read(const char *path, const char *mode, Compress *compression); +extern cfp *cfopen_write(const char *path, const char *mode, Compress *compression); extern int cfread(void *ptr, int size, cfp *fp); extern int cfwrite(const void *ptr, int size, cfp *fp); extern int cfgetc(cfp *fp); @@ -67,4 +61,7 @@ extern int cfclose(cfp *fp); extern int cfeof(cfp *fp); extern const char *get_cfp_error(cfp *fp); +/* also used by tar */ +extern const char * compress_suffix(Compress *compression); + #endif diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 9d0056a569..f2390b7937 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -72,6 +72,25 @@ typedef struct _connParams char *override_dbname; } ConnParams; +typedef enum +{ + COMPR_ALG_DEFAULT = -1, + COMPR_ALG_NONE, + COMPR_ALG_LIBZ, +} CompressionAlgorithm; +/* Should be called "method" or "library" ? */ + +typedef struct Compress { + CompressionAlgorithm alg; + int level; + /* Is a nondefault level set ? This is useful since different compression + * methods have different "default" levels. For now we assume the levels + * are all integer, though. + */ + bool level_set; +} Compress; + + typedef struct _restoreOptions { int createDB; /* Issue commands to create the database */ @@ -125,7 +144,7 @@ typedef struct _restoreOptions int noDataForFailedTables; int exit_on_error; - int compression; + Compress compression; int suppressDumpWarnings; /* Suppress output of WARNING entries * to stderr */ bool single_txn; @@ -281,7 +300,7 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt); /* Create a new archive */ extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker); /* The --list option */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1f82c6499b..3eb6c55600 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -70,7 +70,7 @@ typedef struct _parallelReadyList static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr); static void _getObjectDescription(PQExpBuffer buf, TocEntry *te); static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData); @@ -98,7 +98,7 @@ static int _discoverArchiveFormat(ArchiveHandle *AH); static int RestoringToDB(ArchiveHandle *AH); static void dump_lo_buf(ArchiveHandle *AH); static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim); -static void SetOutput(ArchiveHandle *AH, const char *filename, int compression); +static void SetOutput(ArchiveHandle *AH, const char *filename, Compress *compress); static OutputContext SaveOutput(ArchiveHandle *AH); static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext); @@ -238,7 +238,7 @@ setupRestoreWorker(Archive *AHX) /* Public */ Archive * CreateArchive(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupDumpWorker) { @@ -253,7 +253,9 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt, Archive * OpenArchive(const char *FileSpec, const ArchiveFormat fmt) { - ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker); + Compress compress = {0}; + ArchiveHandle *AH = _allocAH(FileSpec, fmt, &compress, true, archModeRead, + setupRestoreWorker); return (Archive *) AH; } @@ -382,7 +384,7 @@ RestoreArchive(Archive *AHX) * Make sure we won't need (de)compression we haven't got */ #ifndef HAVE_LIBZ - if (AH->compression != 0 && AH->PrintTocDataPtr != NULL) + if (AH->compression.alg != COMPR_ALG_NONE && AH->PrintTocDataPtr != NULL) { for (te = AH->toc->next; te != AH->toc; te = te->next) { @@ -457,8 +459,8 @@ RestoreArchive(Archive *AHX) * Setup the output file if necessary. */ sav = SaveOutput(AH); - if (ropt->filename || ropt->compression) - SetOutput(AH, ropt->filename, ropt->compression); + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) + SetOutput(AH, ropt->filename, &ropt->compression); ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n"); @@ -738,7 +740,7 @@ RestoreArchive(Archive *AHX) */ AH->stage = STAGE_FINALIZING; - if (ropt->filename || ropt->compression) + if (ropt->filename || ropt->compression.alg != COMPR_ALG_NONE) RestoreOutput(AH, sav); if (ropt->useDB) @@ -1123,8 +1125,9 @@ PrintTOCSummary(Archive *AHX) char stamp_str[64]; sav = SaveOutput(AH); + Assert(ropt->compression.alg == COMPR_ALG_NONE); if (ropt->filename) - SetOutput(AH, ropt->filename, 0 /* no compression */ ); + SetOutput(AH, ropt->filename, &ropt->compression); if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT, localtime(&AH->createDate)) == 0) @@ -1133,7 +1136,7 @@ PrintTOCSummary(Archive *AHX) ahprintf(AH, ";\n; Archive created at %s\n", stamp_str); ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n", sanitize_line(AH->archdbname, false), - AH->tocCount, AH->compression); + AH->tocCount, AH->compression.alg); switch (AH->format) { @@ -1487,7 +1490,7 @@ archprintf(Archive *AH, const char *fmt,...) *******************************/ static void -SetOutput(ArchiveHandle *AH, const char *filename, int compression) +SetOutput(ArchiveHandle *AH, const char *filename, Compress *compression) { int fn; @@ -1510,12 +1513,12 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression) /* If compression explicitly requested, use gzopen */ #ifdef HAVE_LIBZ - if (compression != 0) + if (compression->alg != COMPR_ALG_NONE) { char fmode[14]; /* Don't use PG_BINARY_x since this is zlib */ - sprintf(fmode, "wb%d", compression); + sprintf(fmode, "wb%d", compression->level); if (fn >= 0) AH->OF = gzdopen(dup(fn), fmode); else @@ -2259,7 +2262,7 @@ _discoverArchiveFormat(ArchiveHandle *AH) */ static ArchiveHandle * _allocAH(const char *FileSpec, const ArchiveFormat fmt, - const int compression, bool dosync, ArchiveMode mode, + Compress *compression, bool dosync, ArchiveMode mode, SetupWorkerPtrType setupWorkerPtr) { ArchiveHandle *AH; @@ -2310,7 +2313,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, AH->toc->prev = AH->toc; AH->mode = mode; - AH->compression = compression; + AH->compression = *compression; AH->dosync = dosync; memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse)); @@ -2325,7 +2328,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt, * Force stdin/stdout into binary mode if that is what we are using. */ #ifdef WIN32 - if ((fmt != archNull || compression != 0) && + if ((fmt != archNull || compression->alg != COMPR_ALG_NONE) && (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)) { if (mode == archModeWrite) @@ -3741,7 +3744,7 @@ WriteHead(ArchiveHandle *AH) AH->WriteBytePtr(AH, AH->intSize); AH->WriteBytePtr(AH, AH->offSize); AH->WriteBytePtr(AH, AH->format); - WriteInt(AH, AH->compression); + WriteInt(AH, AH->compression.alg); crtm = *localtime(&AH->createDate); WriteInt(AH, crtm.tm_sec); WriteInt(AH, crtm.tm_min); @@ -3816,15 +3819,15 @@ ReadHead(ArchiveHandle *AH) if (AH->version >= K_VERS_1_2) { if (AH->version < K_VERS_1_4) - AH->compression = AH->ReadBytePtr(AH); + AH->compression.alg = AH->ReadBytePtr(AH); else - AH->compression = ReadInt(AH); + AH->compression.alg = ReadInt(AH); } else - AH->compression = Z_DEFAULT_COMPRESSION; + AH->compression.alg = Z_DEFAULT_COMPRESSION; #ifndef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available"); #endif diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a8ea5c7eae..6e033d040e 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -47,7 +47,8 @@ #define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s)) #define GZREAD(p, s, n, fh) fread(p, s, n, fh) #define GZEOF(fh) feof(fh) -/* this is just the redefinition of a libz constant */ +/* this is just the redefinition of a libz constant, in case zlib isn't + * available */ #define Z_DEFAULT_COMPRESSION (-1) typedef struct _z_stream @@ -329,14 +330,7 @@ struct _archiveHandle DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */ struct _tocEntry *currToc; /* Used when dumping data */ - int compression; /*--------- - * Compression requested on open(). - * Possible values for compression: - * -1 Z_DEFAULT_COMPRESSION - * 0 COMPRESSION_NONE - * 1-9 levels for gzip compression - *--------- - */ + Compress compression; /* Compression requested on open */ bool dosync; /* data requested to be synced on sight */ ArchiveMode mode; /* File mode - r or w */ void *formatData; /* Header data specific to file format */ diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c index 77d402c323..55a887a236 100644 --- a/src/bin/pg_dump/pg_backup_custom.c +++ b/src/bin/pg_dump/pg_backup_custom.c @@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) _WriteByte(AH, BLK_DATA); /* Block type */ WriteInt(AH, te->dumpId); /* For sanity check */ - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) WriteInt(AH, oid); - ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc); + ctx->cs = AllocateCompressor(&AH->compression, _CustomWriteFunc); } /* @@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te) static void _PrintData(ArchiveHandle *AH) { - ReadDataFromArchive(AH, AH->compression, _CustomReadFunc); + ReadDataFromArchive(AH, _CustomReadFunc); } static void diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c index 650b542fce..8bf869c6ca 100644 --- a/src/bin/pg_dump/pg_backup_directory.c +++ b/src/bin/pg_dump/pg_backup_directory.c @@ -202,7 +202,7 @@ InitArchiveFmt_Directory(ArchiveHandle *AH) setFilePath(AH, fname, "toc.dat"); - tocFH = cfopen_read(fname, PG_BINARY_R); + tocFH = cfopen_read(fname, PG_BINARY_R, &AH->compression); if (tocFH == NULL) fatal("could not open input file \"%s\": %m", fname); @@ -327,7 +327,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te) setFilePath(AH, fname, tctx->filename); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -388,7 +388,7 @@ _PrintFileData(ArchiveHandle *AH, char *filename) if (!filename) return; - cfp = cfopen_read(filename, PG_BINARY_R); + cfp = cfopen_read(filename, PG_BINARY_R, &AH->compression); if (!cfp) fatal("could not open input file \"%s\": %m", filename); @@ -435,12 +435,13 @@ _LoadBlobs(ArchiveHandle *AH) lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; char line[MAXPGPATH]; + Compress nocompression = {0}; StartRestoreBlobs(AH); setFilePath(AH, fname, "blobs.toc"); - ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R); + ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R, &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open large object TOC file \"%s\" for input: %m", @@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH) { cfp *tocFH; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "toc.dat"); @@ -580,7 +582,7 @@ _CloseArchive(ArchiveHandle *AH) ctx->pstate = ParallelBackupStart(AH); /* The TOC is always created uncompressed */ - tocFH = cfopen_write(fname, PG_BINARY_W, 0); + tocFH = cfopen_write(fname, PG_BINARY_W, &nocompression); if (tocFH == NULL) fatal("could not open output file \"%s\": %m", fname); ctx->dataFH = tocFH; @@ -639,11 +641,12 @@ _StartBlobs(ArchiveHandle *AH, TocEntry *te) { lclContext *ctx = (lclContext *) AH->formatData; char fname[MAXPGPATH]; + Compress nocompression = {0}; setFilePath(AH, fname, "blobs.toc"); /* The blob TOC file is never compressed */ - ctx->blobsTocFH = cfopen_write(fname, "ab", 0); + ctx->blobsTocFH = cfopen_write(fname, "ab", &nocompression); if (ctx->blobsTocFH == NULL) fatal("could not open output file \"%s\": %m", fname); } @@ -661,7 +664,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid); - ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression); + ctx->dataFH = cfopen_write(fname, PG_BINARY_W, &AH->compression); if (ctx->dataFH == NULL) fatal("could not open output file \"%s\": %m", fname); diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c index 61c9c87a9f..4ba79ab924 100644 --- a/src/bin/pg_dump/pg_backup_tar.c +++ b/src/bin/pg_dump/pg_backup_tar.c @@ -39,6 +39,7 @@ #include "pg_backup_archiver.h" #include "pg_backup_tar.h" #include "pg_backup_utils.h" +#include "compress_io.h" #include "pgtar.h" static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te); @@ -196,10 +197,10 @@ InitArchiveFmt_Tar(ArchiveHandle *AH) /* * We don't support compression because reading the files back is not - * possible since gzdopen uses buffered IO which totally screws file + * possible since gzdopen uses buffered IO which totally screws file XXX * positioning. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) fatal("compression is not supported by tar archive format"); } else @@ -254,14 +255,8 @@ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te) ctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry)); if (te->dataDumper != NULL) { -#ifdef HAVE_LIBZ - if (AH->compression == 0) - sprintf(fn, "%d.dat", te->dumpId); - else - sprintf(fn, "%d.dat.gz", te->dumpId); -#else - sprintf(fn, "%d.dat", te->dumpId); -#endif + const char *suffix = compress_suffix(&AH->compression); + sprintf(fn, "%d.dat%s", te->dumpId, suffix); ctx->filename = pg_strdup(fn); } else @@ -352,7 +347,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression == 0) + if (AH->compression.alg == COMPR_ALG_NONE) tm->nFH = ctx->tarFH; else fatal("compression is not supported by tar archive format"); @@ -413,9 +408,9 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode) #ifdef HAVE_LIBZ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) { - sprintf(fmode, "wb%d", AH->compression); + sprintf(fmode, "wb%d", AH->compression.level); tm->zFH = gzdopen(dup(fileno(tm->tmpFH)), fmode); if (tm->zFH == NULL) fatal("could not open temporary file"); @@ -443,7 +438,7 @@ tarClose(ArchiveHandle *AH, TAR_MEMBER *th) /* * Close the GZ file since we dup'd. This will flush the buffers. */ - if (AH->compression != 0) + if (AH->compression.alg != COMPR_ALG_NONE) if (GZCLOSE(th->zFH) != 0) fatal("could not close tar member"); @@ -868,7 +863,7 @@ _CloseArchive(ArchiveHandle *AH) memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions)); ropt->filename = NULL; ropt->dropSchema = 1; - ropt->compression = 0; + ropt->compression.alg = COMPR_ALG_NONE; ropt->superuser = NULL; ropt->suppressDumpWarnings = true; @@ -952,16 +947,12 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid) lclContext *ctx = (lclContext *) AH->formatData; lclTocEntry *tctx = (lclTocEntry *) te->formatData; char fname[255]; - char *sfx; + const char *sfx; if (oid == 0) fatal("invalid OID for large object (%u)", oid); - if (AH->compression != 0) - sfx = ".gz"; - else - sfx = ""; - + sfx = compress_suffix(&AH->compression); sprintf(fname, "blob_%u.dat%s", oid, sfx); tarPrintf(ctx->blobToc, "%u %s\n", oid, fname); diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1ab98a2286..4cbc79aedc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -59,6 +59,7 @@ #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" +#include "compress_io.h" #include "pg_backup_db.h" #include "pg_backup_utils.h" #include "pg_dump.h" @@ -297,6 +298,82 @@ static void setupDumpWorker(Archive *AHX); static TableInfo *getRootTableInfo(TableInfo *tbinfo); +/* Parse the string into compression options */ +static void +parse_compression(const char *optarg, Compress *compress) +{ + if (optarg[0] == '0' && optarg[1] == '\0') + compress->alg = COMPR_ALG_NONE; + else if ((optarg[0] > '0' && optarg[0] <= '9') || + optarg[0] == '-') + { + compress->alg = COMPR_ALG_LIBZ; + compress->level_set = true; + compress->level = atoi(optarg); + if (optarg[1] != '\0') + { + pg_log_error("compression level must be in range 0..9"); + exit_nicely(1); + } + } + else + { + /* Parse a more flexible string like level=3 alg=zlib opts=long */ + for (;;) + { + char *eq = strchr(optarg, '='); + int len; + + if (eq == NULL) + { + pg_log_error("compression options must be key=value: %s", optarg); + exit_nicely(1); + } + + len = eq - optarg; + if (strncmp(optarg, "alg", len) == 0) + { + if (strchr(eq, ' ')) + len = strchr(eq, ' ') - eq - 1; + else + len = strlen(eq) - len; + if (strncmp(1+eq, "zlib", len) == 0 || + strncmp(1+eq, "libz", len) == 0) + compress->alg = COMPR_ALG_LIBZ; + else + { + pg_log_error("unknown compression algorithm: %s", 1+eq); + exit_nicely(1); + } + } + else if (strncmp(optarg, "level", len) == 0) + { + compress->level = atoi(1+eq); + compress->level_set = true; + } + else + { + pg_log_error("unknown compression setting: %s", optarg); + exit_nicely(1); + } + + optarg = strchr(eq, ' '); + if (!optarg++) + break; + } + + if (!compress->level_set) + { + const int default_compress_level[] = { + 0, /* COMPR_ALG_NONE */ + Z_DEFAULT_COMPRESSION, /* COMPR_ALG_ZLIB */ + }; + + compress->level = default_compress_level[compress->alg]; + } + } +} + int main(int argc, char **argv) { @@ -319,7 +396,7 @@ main(int argc, char **argv) char *use_role = NULL; long rowsPerInsert; int numWorkers = 1; - int compressLevel = -1; + Compress compress = { .alg = COMPR_ALG_DEFAULT }; int plainText = 0; ArchiveFormat archiveFormat = archUnknown; ArchiveMode archiveMode; @@ -532,12 +609,7 @@ main(int argc, char **argv) break; case 'Z': /* Compression Level */ - compressLevel = atoi(optarg); - if (compressLevel < 0 || compressLevel > 9) - { - pg_log_error("compression level must be in range 0..9"); - exit_nicely(1); - } + parse_compression(optarg, &compress); break; case 0: @@ -679,20 +751,28 @@ main(int argc, char **argv) plainText = 1; /* Custom and directory formats are compressed by default, others not */ - if (compressLevel == -1) + if (compress.alg == COMPR_ALG_DEFAULT) { -#ifdef HAVE_LIBZ if (archiveFormat == archCustom || archiveFormat == archDirectory) - compressLevel = Z_DEFAULT_COMPRESSION; - else + { +#ifdef HAVE_LIBZ + compress.alg = COMPR_ALG_LIBZ; + compress.level = Z_DEFAULT_COMPRESSION; #endif - compressLevel = 0; + } + else + { + compress.alg = COMPR_ALG_NONE; + compress.level = 0; + } } #ifndef HAVE_LIBZ - if (compressLevel != 0) + if (compress.alg == COMPR_ALG_LIBZ) + { pg_log_warning("requested compression not available in this installation -- archive will be uncompressed"); - compressLevel = 0; + compress.alg = 0; + } #endif /* @@ -723,7 +803,7 @@ main(int argc, char **argv) fatal("option --index-collation-versions-unknown only works in binary upgrade mode"); /* Open the output file */ - fout = CreateArchive(filename, archiveFormat, compressLevel, dosync, + fout = CreateArchive(filename, archiveFormat, &compress, dosync, archiveMode, setupDumpWorker); /* Make dump options accessible right away */ @@ -957,10 +1037,7 @@ main(int argc, char **argv) ropt->sequence_data = dopt.sequence_data; ropt->binary_upgrade = dopt.binary_upgrade; - if (compressLevel == -1) - ropt->compression = 0; - else - ropt->compression = compressLevel; + ropt->compression = compress; ropt->suppressDumpWarnings = true; /* We've already shown them */ -- 2.17.0 --GRPZ8SYKNexpdSJ7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-struct-compressLibs.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v4 1/2] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 176 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 23 +++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 +++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 ++++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 75 ++++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 20 files changed, 386 insertions(+) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..2b2be4bb99 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,176 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char key[64]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +static void init_dsm_registry(void); + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a DSM entry. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(key); + Assert(size); + Assert(found); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find_or_insert(dsm_registry_table, key_padded, found); + if (!(*found)) + { + /* Initialize DSM registry entry. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing DSM registry entry. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of already-attached DSM registry entry. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 706140eb9f..44d6a243e5 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 79a16d077f..88fef448be 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..f3faa991d1 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f61ec3e59d..f13077bd8c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -325,6 +325,7 @@ WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consu NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." WALSummarizer "Waiting to read or update WAL summarization state." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." XactBuffer "Waiting for I/O on a transaction status SLRU buffer." CommitTsBuffer "Waiting for I/O on a commit timestamp SLRU buffer." @@ -355,6 +356,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..247ac0acc1 --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void *dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..e0e7b3b765 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index b76f588559..bd53d52a3f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..4a7992109b --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..068f47ed02 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,75 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +init_state(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +dsm_registry_attach(void) +{ + bool found; + + tdr_state = dsm_registry_init_or_attach("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + init_state, &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5fd46b7bd1..469f7570f5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2799,6 +2801,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --BOKacYhQ+x31HxR3 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v5 1/2] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 176 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 23 +++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 +++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 ++++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 75 ++++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 20 files changed, 386 insertions(+) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..3a32dcfe40 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,176 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char key[64]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +static void init_dsm_registry(void); + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a DSM entry. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(key); + Assert(size); + Assert(found); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find_or_insert(dsm_registry_table, key_padded, found); + if (!(*found)) + { + /* Initialize DSM registry entry. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing DSM registry entry. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of an already-attached DSM registry entry. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index e5119ed55d..fbc62b1563 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 08bdc718b8..5a936171f7 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index b4b989ac56..2f2de5a562 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index 088eb977d4..33c4c645b8 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -325,6 +325,7 @@ WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consu NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." WALSummarizer "Waiting to read or update WAL summarization state." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." XactBuffer "Waiting for I/O on a transaction status SLRU buffer." CommitTsBuffer "Waiting for I/O on a commit timestamp SLRU buffer." @@ -355,6 +356,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..5d4cf9ca49 --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void *dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 167ae34208..50a65e046d 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 00ff1d77d1..2c3b8d73bc 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..a4045fea37 --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..b57f13cebb --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,75 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +init_state(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +dsm_registry_attach(void) +{ + bool found; + + tdr_state = dsm_registry_init_or_attach("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + init_state, &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5fd46b7bd1..469f7570f5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2799,6 +2801,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --dTy3Mrz/UPE2dbVg Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v6 2/3] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- doc/src/sgml/xfunc.sgml | 49 ++++- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 174 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 23 +++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 +++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 ++++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 76 ++++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 21 files changed, 431 insertions(+), 3 deletions(-) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 0ba52b41d4..9d305000f9 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -3460,6 +3460,45 @@ LWLockRelease(AddinShmemInitLock); the <productname>PostgreSQL</productname> source tree. </para> </sect3> + + <sect3 id="xfunc-shared-addin-after-startup"> + <title>Requesting Shared Memory After Startup</title> + + <para> + There is another, more flexible method of reserving shared memory that + can be done after server startup and outside a + <literal>shmem_request_hook</literal>. To do so, each backend that will + use the shared memory should obtain a pointer to it by calling: +<programlisting> +void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found) +</programlisting> + If a dynamic shared memory segment with the given name does not yet + exist, this function will allocate it and initialize it with the provided + <function>init_callback</function> callback function. If the segment has + already been allocated and initialized by another backend, this function + simply attaches the existing dynamic shared memory segment to the current + backend. + </para> + + <para> + Unlike shared memory reserved at server startup, there is no need to + acquire <function>AddinShmemInitLock</function> or otherwise take action + to avoid race conditions when reserving shared memory with + <function>GetNamedDSMSegment</function>. This function ensures that only + one backend allocates and initializes the segment and that all other + backends receive a pointer to the fully allocated and initialized + segment. + </para> + + <para> + A complete usage example of <function>GetNamedDSMSegment</function> can + be found in + <filename>src/test/modules/test_dsm_registry/test_dsm_registry.c</filename> + in the <productname>PostgreSQL</productname> source tree. + </para> + </sect3> </sect2> <sect2 id="xfunc-addin-lwlocks"> @@ -3469,8 +3508,9 @@ LWLockRelease(AddinShmemInitLock); <title>Requesting LWLocks at Startup</title> <para> - Add-ins can reserve LWLocks on server startup. Like with shared memory, - the add-in's shared library must be preloaded by specifying it in + Add-ins can reserve LWLocks on server startup. Like with shared memory + reserved at server startup, the add-in's shared library must be preloaded + by specifying it in <xref linkend="guc-shared-preload-libraries"/><indexterm><primary>shared_preload_libraries</primary></indexterm>, and the shared library should register a <literal>shmem_request_hook</literal> in its @@ -3508,7 +3548,10 @@ void LWLockInitialize(LWLock *lock, int tranche_id) process allocates a new <literal>tranche_id</literal> and initializes each new LWLock. One way to do this is to only call these functions in your shared memory initialization code with the - <function>AddinShmemInitLock</function> held exclusively. + <function>AddinShmemInitLock</function> held exclusively. If using + <function>GetNamedDSMSegment</function>, calling these functions in the + <function>init_callback</function> callback function is sufficient to + avoid race conditions. </para> <para> diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..7deaa0d6c5 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,174 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char name[64]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a named DSM segment. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char name_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(name); + Assert(size); + Assert(found); + + if (strlen(name) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM segment name too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(name_padded, name); + entry = dshash_find_or_insert(dsm_registry_table, name_padded, found); + if (!(*found)) + { + /* Initialize the segment. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing segment. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of an already-attached segment. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index e5119ed55d..fbc62b1563 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 08bdc718b8..5a936171f7 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index b4b989ac56..2f2de5a562 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f625473ad4..6bcb1cca0c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -329,6 +329,7 @@ WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consu NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." WALSummarizer "Waiting to read or update WAL summarization state." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." # # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE) @@ -367,6 +368,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..8e8a23ba60 --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 167ae34208..50a65e046d 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 00ff1d77d1..2c3b8d73bc 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..a4045fea37 --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..b46cd6c620 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,76 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +tdr_init_shmem(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +tdr_attach_shmem(void) +{ + bool found; + + tdr_state = GetNamedDSMSegment("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + tdr_init_shmem, + &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f582eb59e7..b7e73736e9 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2799,6 +2801,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --FCuugMFkClbJLl1L Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0003-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v7 2/3] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- doc/src/sgml/xfunc.sgml | 49 ++++- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 174 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 23 +++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 +++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 ++++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 76 ++++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 21 files changed, 431 insertions(+), 3 deletions(-) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index 82e1dadcca..fbeac2fd1d 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -3460,6 +3460,45 @@ LWLockRelease(AddinShmemInitLock); the <productname>PostgreSQL</productname> source tree. </para> </sect3> + + <sect3 id="xfunc-shared-addin-after-startup"> + <title>Requesting Shared Memory After Startup</title> + + <para> + There is another, more flexible method of reserving shared memory that + can be done after server startup and outside a + <literal>shmem_request_hook</literal>. To do so, each backend that will + use the shared memory should obtain a pointer to it by calling: +<programlisting> +void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found) +</programlisting> + If a dynamic shared memory segment with the given name does not yet + exist, this function will allocate it and initialize it with the provided + <function>init_callback</function> callback function. If the segment has + already been allocated and initialized by another backend, this function + simply attaches the existing dynamic shared memory segment to the current + backend. + </para> + + <para> + Unlike shared memory reserved at server startup, there is no need to + acquire <function>AddinShmemInitLock</function> or otherwise take action + to avoid race conditions when reserving shared memory with + <function>GetNamedDSMSegment</function>. This function ensures that only + one backend allocates and initializes the segment and that all other + backends receive a pointer to the fully allocated and initialized + segment. + </para> + + <para> + A complete usage example of <function>GetNamedDSMSegment</function> can + be found in + <filename>src/test/modules/test_dsm_registry/test_dsm_registry.c</filename> + in the <productname>PostgreSQL</productname> source tree. + </para> + </sect3> </sect2> <sect2 id="xfunc-addin-lwlocks"> @@ -3469,8 +3508,9 @@ LWLockRelease(AddinShmemInitLock); <title>Requesting LWLocks at Startup</title> <para> - Add-ins can reserve LWLocks on server startup. As with shared memory, - the add-in's shared library must be preloaded by specifying it in + Add-ins can reserve LWLocks on server startup. As with shared memory + reserved at server startup, the add-in's shared library must be preloaded + by specifying it in <xref linkend="guc-shared-preload-libraries"/><indexterm><primary>shared_preload_libraries</primary></indexterm>, and the shared library should register a <literal>shmem_request_hook</literal> in its @@ -3508,7 +3548,10 @@ void LWLockInitialize(LWLock *lock, int tranche_id) process allocates a new <literal>tranche_id</literal> and initializes each new LWLock. One way to do this is to only call these functions in your shared memory initialization code with the - <function>AddinShmemInitLock</function> held exclusively. + <function>AddinShmemInitLock</function> held exclusively. If using + <function>GetNamedDSMSegment</function>, calling these functions in the + <function>init_callback</function> callback function is sufficient to + avoid race conditions. </para> <para> diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..7deaa0d6c5 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,174 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char name[64]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a named DSM segment. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char name_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(name); + Assert(size); + Assert(found); + + if (strlen(name) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM segment name too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(name_padded, name); + entry = dshash_find_or_insert(dsm_registry_table, name_padded, found); + if (!(*found)) + { + /* Initialize the segment. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing segment. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of an already-attached segment. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index e5119ed55d..fbc62b1563 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 08bdc718b8..5a936171f7 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index b4b989ac56..2f2de5a562 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f625473ad4..6bcb1cca0c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -329,6 +329,7 @@ WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consu NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." WALSummarizer "Waiting to read or update WAL summarization state." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." # # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE) @@ -367,6 +368,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..8e8a23ba60 --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 167ae34208..50a65e046d 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 00ff1d77d1..2c3b8d73bc 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..a4045fea37 --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..b46cd6c620 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,76 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +tdr_init_shmem(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +tdr_attach_shmem(void) +{ + bool found; + + tdr_state = GetNamedDSMSegment("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + tdr_init_shmem, + &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f582eb59e7..b7e73736e9 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2799,6 +2801,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --huq684BweRXVnRxX Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0003-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v8 2/3] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- doc/src/sgml/xfunc.sgml | 49 ++++- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 178 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 23 +++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 +++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 ++++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 76 ++++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 21 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/doc/src/sgml/xfunc.sgml b/doc/src/sgml/xfunc.sgml index ede2a5dea6..0ad9f38e90 100644 --- a/doc/src/sgml/xfunc.sgml +++ b/doc/src/sgml/xfunc.sgml @@ -3460,6 +3460,45 @@ LWLockRelease(AddinShmemInitLock); the <productname>PostgreSQL</productname> source tree. </para> </sect3> + + <sect3 id="xfunc-shared-addin-after-startup"> + <title>Requesting Shared Memory After Startup</title> + + <para> + There is another, more flexible method of reserving shared memory that + can be done after server startup and outside a + <literal>shmem_request_hook</literal>. To do so, each backend that will + use the shared memory should obtain a pointer to it by calling: +<programlisting> +void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found) +</programlisting> + If a dynamic shared memory segment with the given name does not yet + exist, this function will allocate it and initialize it with the provided + <function>init_callback</function> callback function. If the segment has + already been allocated and initialized by another backend, this function + simply attaches the existing dynamic shared memory segment to the current + backend. + </para> + + <para> + Unlike shared memory reserved at server startup, there is no need to + acquire <function>AddinShmemInitLock</function> or otherwise take action + to avoid race conditions when reserving shared memory with + <function>GetNamedDSMSegment</function>. This function ensures that only + one backend allocates and initializes the segment and that all other + backends receive a pointer to the fully allocated and initialized + segment. + </para> + + <para> + A complete usage example of <function>GetNamedDSMSegment</function> can + be found in + <filename>src/test/modules/test_dsm_registry/test_dsm_registry.c</filename> + in the <productname>PostgreSQL</productname> source tree. + </para> + </sect3> </sect2> <sect2 id="xfunc-addin-lwlocks"> @@ -3469,8 +3508,9 @@ LWLockRelease(AddinShmemInitLock); <title>Requesting LWLocks at Startup</title> <para> - Add-ins can reserve LWLocks on server startup. As with shared memory, - the add-in's shared library must be preloaded by specifying it in + Add-ins can reserve LWLocks on server startup. As with shared memory + reserved at server startup, the add-in's shared library must be preloaded + by specifying it in <xref linkend="guc-shared-preload-libraries"/><indexterm><primary>shared_preload_libraries</primary></indexterm>, and the shared library should register a <literal>shmem_request_hook</literal> in its @@ -3508,7 +3548,10 @@ void LWLockInitialize(LWLock *lock, int tranche_id) process allocates a new <literal>tranche_id</literal> and initializes each new LWLock. One way to do this is to only call these functions in your shared memory initialization code with the - <function>AddinShmemInitLock</function> held exclusively. + <function>AddinShmemInitLock</function> held exclusively. If using + <function>GetNamedDSMSegment</function>, calling these functions in the + <function>init_callback</function> callback function is sufficient to + avoid race conditions. </para> <para> diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..89f7d670e4 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,178 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char name[64]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a named DSM segment. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char name_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(found); + + if (!name || *name == '\0') + elog(ERROR, "DSM segment name cannot be empty"); + + if (strlen(name) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM segment name too long"); + + if (size == 0) + elog(ERROR, "DSM segment size must be nonzero"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(name_padded, name); + entry = dshash_find_or_insert(dsm_registry_table, name_padded, found); + if (!(*found)) + { + /* Initialize the segment. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing segment. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of an already-attached segment. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index e5119ed55d..fbc62b1563 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 08bdc718b8..5a936171f7 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index b4b989ac56..2f2de5a562 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index f625473ad4..6bcb1cca0c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -329,6 +329,7 @@ WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consu NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." WALSummarizer "Waiting to read or update WAL summarization state." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." # # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE) @@ -367,6 +368,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..8e8a23ba60 --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void *GetNamedDSMSegment(const char *name, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 167ae34208..50a65e046d 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 00ff1d77d1..2c3b8d73bc 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..a4045fea37 --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..b46cd6c620 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,76 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +tdr_init_shmem(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +tdr_attach_shmem(void) +{ + bool found; + + tdr_state = GetNamedDSMSegment("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + tdr_init_shmem, + &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + tdr_attach_shmem(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f582eb59e7..b7e73736e9 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2799,6 +2801,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --hHWLQfXTYDoKhP50 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0003-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v1 1/2] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 176 +++++++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + src/include/storage/dsm_registry.h | 22 +++ src/include/storage/lwlock.h | 4 +- src/tools/pgindent/typedefs.list | 2 + 9 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..ea80f45716 --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,176 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char key[256]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +static void init_dsm_registry(void); + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Initialize or attach a DSM entry. + * + * *ptr should initially be set to NULL. If it is not NULL, this routine will + * assume that the segment has already been attached to the current session. + * Otherwise, this routine will set *ptr appropriately. + * + * init_callback is called to initialize the segment when it is first created. + */ +void +dsm_registry_init_or_attach(const char *key, void **ptr, size_t size, + void (*init_callback) (void *ptr)) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + bool found; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + + Assert(key); + Assert(ptr); + Assert(size); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Quick exit if the value is already set. */ + if (*ptr) + return; + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find_or_insert(dsm_registry_table, key_padded, &found); + if (!found) + { + /* Initialize DSM registry entry. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + *ptr = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (*ptr); + } + else + { + /* Attach to existing DSM registry entry. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + *ptr = dsm_segment_address(seg); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 2225a4a6e6..034b656115 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -38,6 +38,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -113,6 +114,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -285,6 +287,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 79a16d077f..88fef448be 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..f3faa991d1 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index f72f2906ce..e8f679c8ae 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -54,3 +54,4 @@ XactTruncationLock 44 WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 +DSMRegistryLock 49 diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..8c311e50ae --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern void dsm_registry_init_or_attach(const char *key, void **ptr, size_t size, + void (*init_callback) (void *ptr)); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..665d471418 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,7 +207,9 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, - LWTRANCHE_FIRST_USER_DEFINED, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, + LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; /* diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index d659adbfd6..c89a268d9e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -606,6 +606,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions -- 2.25.1 --DocE+STaALJfprDB Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v1-0002-test-dsm-registry.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v2 1/2] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 209 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 24 ++ src/include/storage/lwlock.h | 4 +- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 ++ .../expected/test_dsm_registry.out | 16 ++ .../modules/test_dsm_registry/meson.build | 33 +++ .../sql/test_dsm_registry.sql | 7 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 75 +++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 20 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..5fc970001e --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,209 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char key[256]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +static void init_dsm_registry(void); + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Find an existing DSM entry. + * + * This routine returns whether a segment is registered with the provided key. + */ +bool +dsm_registry_find(const char *key) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + + Assert(key); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find(dsm_registry_table, key_padded, false); + if (entry) + dshash_release_lock(dsm_registry_table, entry); + + MemoryContextSwitchTo(oldcontext); + + return entry != NULL; +} + +/* + * Initialize or attach a DSM entry. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(key); + Assert(size); + Assert(found); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find_or_insert(dsm_registry_table, key_padded, found); + if (!(*found)) + { + /* Initialize DSM registry entry. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing DSM registry entry. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of already-attached DSM registry entry. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 706140eb9f..44d6a243e5 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 79a16d077f..88fef448be 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..f3faa991d1 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index 7e79163466..de1bc986e1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -324,6 +324,7 @@ XactTruncation "Waiting to execute <function>pg_xact_status</function> or update WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consumption." NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." XactBuffer "Waiting for I/O on a transaction status SLRU buffer." CommitTsBuffer "Waiting for I/O on a commit timestamp SLRU buffer." @@ -354,6 +355,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..5e9c1000bc --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern bool dsm_registry_find(const char *key); +extern void *dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..665d471418 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,7 +207,9 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, - LWTRANCHE_FIRST_USER_DEFINED, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, + LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; /* diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index b76f588559..bd53d52a3f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..b9746f8a1e --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,16 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +CREATE DATABASE test; +\c test +CREATE EXTENSION test_dsm_registry; +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..4a7992109b --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..bf2f468f10 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,7 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + +CREATE DATABASE test; +\c test +CREATE EXTENSION test_dsm_registry; +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..068f47ed02 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,75 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +init_state(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +dsm_registry_attach(void) +{ + bool found; + + tdr_state = dsm_registry_init_or_attach("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + init_state, &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e37ef9aa76..4270109fb8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2796,6 +2798,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --FCuugMFkClbJLl1L Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v2-0002-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
* [PATCH v3 1/2] add dsm registry @ 2023-10-12 03:07 Nathan Bossart <nathan@postgresql.org> 0 siblings, 0 replies; 9+ messages in thread From: Nathan Bossart @ 2023-10-12 03:07 UTC (permalink / raw) --- src/backend/storage/ipc/Makefile | 1 + src/backend/storage/ipc/dsm_registry.c | 209 ++++++++++++++++++ src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/ipc/meson.build | 1 + src/backend/storage/lmgr/lwlock.c | 4 + src/backend/storage/lmgr/lwlocknames.txt | 1 + .../utils/activity/wait_event_names.txt | 3 + src/include/storage/dsm_registry.h | 24 ++ src/include/storage/lwlock.h | 2 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_dsm_registry/.gitignore | 4 + src/test/modules/test_dsm_registry/Makefile | 23 ++ .../expected/test_dsm_registry.out | 14 ++ .../modules/test_dsm_registry/meson.build | 33 +++ .../sql/test_dsm_registry.sql | 4 + .../test_dsm_registry--1.0.sql | 10 + .../test_dsm_registry/test_dsm_registry.c | 75 +++++++ .../test_dsm_registry.control | 4 + src/tools/pgindent/typedefs.list | 3 + 20 files changed, 420 insertions(+) create mode 100644 src/backend/storage/ipc/dsm_registry.c create mode 100644 src/include/storage/dsm_registry.h create mode 100644 src/test/modules/test_dsm_registry/.gitignore create mode 100644 src/test/modules/test_dsm_registry/Makefile create mode 100644 src/test/modules/test_dsm_registry/expected/test_dsm_registry.out create mode 100644 src/test/modules/test_dsm_registry/meson.build create mode 100644 src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.c create mode 100644 src/test/modules/test_dsm_registry/test_dsm_registry.control diff --git a/src/backend/storage/ipc/Makefile b/src/backend/storage/ipc/Makefile index 6d5b921038..d8a1653eb6 100644 --- a/src/backend/storage/ipc/Makefile +++ b/src/backend/storage/ipc/Makefile @@ -12,6 +12,7 @@ OBJS = \ barrier.o \ dsm.o \ dsm_impl.o \ + dsm_registry.o \ ipc.o \ ipci.o \ latch.o \ diff --git a/src/backend/storage/ipc/dsm_registry.c b/src/backend/storage/ipc/dsm_registry.c new file mode 100644 index 0000000000..5fc970001e --- /dev/null +++ b/src/backend/storage/ipc/dsm_registry.c @@ -0,0 +1,209 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.c + * + * Functions for interfacing with the dynamic shared memory registry. This + * provides a way for libraries to use shared memory without needing to + * request it at startup time via a shmem_request_hook. + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/storage/ipc/dsm_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "lib/dshash.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/memutils.h" + +typedef struct DSMRegistryCtxStruct +{ + dsa_handle dsah; + dshash_table_handle dshh; +} DSMRegistryCtxStruct; + +static DSMRegistryCtxStruct *DSMRegistryCtx; + +typedef struct DSMRegistryEntry +{ + char key[256]; + dsm_handle handle; +} DSMRegistryEntry; + +static const dshash_parameters dsh_params = { + offsetof(DSMRegistryEntry, handle), + sizeof(DSMRegistryEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_DSM_REGISTRY_HASH +}; + +static dsa_area *dsm_registry_dsa; +static dshash_table *dsm_registry_table; + +static void init_dsm_registry(void); + +Size +DSMRegistryShmemSize(void) +{ + return MAXALIGN(sizeof(DSMRegistryCtxStruct)); +} + +void +DSMRegistryShmemInit(void) +{ + bool found; + + DSMRegistryCtx = (DSMRegistryCtxStruct *) + ShmemInitStruct("DSM Registry Data", + DSMRegistryShmemSize(), + &found); + + if (!found) + { + DSMRegistryCtx->dsah = DSA_HANDLE_INVALID; + DSMRegistryCtx->dshh = DSHASH_HANDLE_INVALID; + } +} + +/* + * Initialize or attach to the dynamic shared hash table that stores the DSM + * registry entries, if not already done. This must be called before accessing + * the table. + */ +static void +init_dsm_registry(void) +{ + /* Quick exit if we already did this. */ + if (dsm_registry_table) + return; + + /* Otherwise, use a lock to ensure only one process creates the table. */ + LWLockAcquire(DSMRegistryLock, LW_EXCLUSIVE); + + if (DSMRegistryCtx->dshh == DSHASH_HANDLE_INVALID) + { + /* Initialize dynamic shared hash table for registry. */ + dsm_registry_dsa = dsa_create(LWTRANCHE_DSM_REGISTRY_DSA); + dsa_pin(dsm_registry_dsa); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_create(dsm_registry_dsa, &dsh_params, 0); + + /* Store handles in shared memory for other backends to use. */ + DSMRegistryCtx->dsah = dsa_get_handle(dsm_registry_dsa); + DSMRegistryCtx->dshh = dshash_get_hash_table_handle(dsm_registry_table); + } + else + { + /* Attach to existing dynamic shared hash table. */ + dsm_registry_dsa = dsa_attach(DSMRegistryCtx->dsah); + dsa_pin_mapping(dsm_registry_dsa); + dsm_registry_table = dshash_attach(dsm_registry_dsa, &dsh_params, + DSMRegistryCtx->dshh, 0); + } + + LWLockRelease(DSMRegistryLock); +} + +/* + * Find an existing DSM entry. + * + * This routine returns whether a segment is registered with the provided key. + */ +bool +dsm_registry_find(const char *key) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + + Assert(key); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find(dsm_registry_table, key_padded, false); + if (entry) + dshash_release_lock(dsm_registry_table, entry); + + MemoryContextSwitchTo(oldcontext); + + return entry != NULL; +} + +/* + * Initialize or attach a DSM entry. + * + * This routine returns the address of the segment. init_callback is called to + * initialize the segment when it is first created. + */ +void * +dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), bool *found) +{ + DSMRegistryEntry *entry; + MemoryContext oldcontext; + char key_padded[offsetof(DSMRegistryEntry, handle)] = {0}; + void *ret; + + Assert(key); + Assert(size); + Assert(found); + + if (strlen(key) >= offsetof(DSMRegistryEntry, handle)) + elog(ERROR, "DSM registry key too long"); + + /* Be sure any local memory allocated by DSM/DSA routines is persistent. */ + oldcontext = MemoryContextSwitchTo(TopMemoryContext); + + /* Connect to the registry. */ + init_dsm_registry(); + + strcpy(key_padded, key); + entry = dshash_find_or_insert(dsm_registry_table, key_padded, found); + if (!(*found)) + { + /* Initialize DSM registry entry. */ + dsm_segment *seg = dsm_create(size, 0); + + dsm_pin_segment(seg); + dsm_pin_mapping(seg); + entry->handle = dsm_segment_handle(seg); + ret = dsm_segment_address(seg); + + if (init_callback) + (*init_callback) (ret); + } + else if (!dsm_find_mapping(entry->handle)) + { + /* Attach to existing DSM registry entry. */ + dsm_segment *seg = dsm_attach(entry->handle); + + dsm_pin_mapping(seg); + ret = dsm_segment_address(seg); + } + else + { + /* Return address of already-attached DSM registry entry. */ + ret = dsm_segment_address(dsm_find_mapping(entry->handle)); + } + + dshash_release_lock(dsm_registry_table, entry); + MemoryContextSwitchTo(oldcontext); + + return ret; +} diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 706140eb9f..44d6a243e5 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -40,6 +40,7 @@ #include "replication/walsender.h" #include "storage/bufmgr.h" #include "storage/dsm.h" +#include "storage/dsm_registry.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" #include "storage/pmsignal.h" @@ -115,6 +116,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt))); size = add_size(size, dsm_estimate_size()); + size = add_size(size, DSMRegistryShmemSize()); size = add_size(size, BufferShmemSize()); size = add_size(size, LockShmemSize()); size = add_size(size, PredicateLockShmemSize()); @@ -289,6 +291,7 @@ CreateOrAttachShmemStructs(void) InitShmemIndex(); dsm_shmem_init(); + DSMRegistryShmemInit(); /* * Set up xlog, clog, and buffers diff --git a/src/backend/storage/ipc/meson.build b/src/backend/storage/ipc/meson.build index 79a16d077f..88fef448be 100644 --- a/src/backend/storage/ipc/meson.build +++ b/src/backend/storage/ipc/meson.build @@ -4,6 +4,7 @@ backend_sources += files( 'barrier.c', 'dsm.c', 'dsm_impl.c', + 'dsm_registry.c', 'ipc.c', 'ipci.c', 'latch.c', diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 315a78cda9..f3faa991d1 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -190,6 +190,10 @@ static const char *const BuiltinTrancheNames[] = { "LogicalRepLauncherDSA", /* LWTRANCHE_LAUNCHER_HASH: */ "LogicalRepLauncherHash", + /* LWTRANCHE_DSM_REGISTRY_DSA: */ + "DSMRegistryDSA", + /* LWTRANCHE_DSM_REGISTRY_HASH: */ + "DSMRegistryHash", }; StaticAssertDecl(lengthof(BuiltinTrancheNames) == diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index d621f5507f..ef8542de46 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -55,3 +55,4 @@ WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 WaitEventExtensionLock 48 WALSummarizerLock 49 +DSMRegistryLock 50 diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index 7e79163466..de1bc986e1 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -324,6 +324,7 @@ XactTruncation "Waiting to execute <function>pg_xact_status</function> or update WrapLimitsVacuum "Waiting to update limits on transaction id and multixact consumption." NotifyQueueTail "Waiting to update limit on <command>NOTIFY</command> message storage." WaitEventExtension "Waiting to read or update custom wait events information for extensions." +DSMRegistry "Waiting to read or update the dynamic shared memory registry." XactBuffer "Waiting for I/O on a transaction status SLRU buffer." CommitTsBuffer "Waiting for I/O on a commit timestamp SLRU buffer." @@ -354,6 +355,8 @@ PgStatsHash "Waiting for stats shared memory hash table access." PgStatsData "Waiting for shared memory stats data access." LogicalRepLauncherDSA "Waiting to access logical replication launcher's dynamic shared memory allocator." LogicalRepLauncherHash "Waiting to access logical replication launcher's shared hash table." +DSMRegistryDSA "Waiting to access dynamic shared memory registry's dynamic shared memory allocator." +DSMRegistryHash "Waiting to access dynamic shared memory registry's shared hash table." # # Wait Events - Lock diff --git a/src/include/storage/dsm_registry.h b/src/include/storage/dsm_registry.h new file mode 100644 index 0000000000..5e9c1000bc --- /dev/null +++ b/src/include/storage/dsm_registry.h @@ -0,0 +1,24 @@ +/*------------------------------------------------------------------------- + * + * dsm_registry.h + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/dsm_registry.h + * + *------------------------------------------------------------------------- + */ +#ifndef DSM_REGISTRY_H +#define DSM_REGISTRY_H + +extern bool dsm_registry_find(const char *key); +extern void *dsm_registry_init_or_attach(const char *key, size_t size, + void (*init_callback) (void *ptr), + bool *found); + +extern Size DSMRegistryShmemSize(void); +extern void DSMRegistryShmemInit(void); + +#endif /* DSM_REGISTRY_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index b038e599c0..e0e7b3b765 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -207,6 +207,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_PGSTATS_DATA, LWTRANCHE_LAUNCHER_DSA, LWTRANCHE_LAUNCHER_HASH, + LWTRANCHE_DSM_REGISTRY_DSA, + LWTRANCHE_DSM_REGISTRY_HASH, LWTRANCHE_FIRST_USER_DEFINED, } BuiltinTrancheIds; diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 5d33fa6a9a..f656032589 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ test_custom_rmgrs \ test_ddl_deparse \ test_dsa \ + test_dsm_registry \ test_extensions \ test_ginpostinglist \ test_integerset \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index b76f588559..bd53d52a3f 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -15,6 +15,7 @@ subdir('test_copy_callbacks') subdir('test_custom_rmgrs') subdir('test_ddl_deparse') subdir('test_dsa') +subdir('test_dsm_registry') subdir('test_extensions') subdir('test_ginpostinglist') subdir('test_integerset') diff --git a/src/test/modules/test_dsm_registry/.gitignore b/src/test/modules/test_dsm_registry/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/test_dsm_registry/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_dsm_registry/Makefile b/src/test/modules/test_dsm_registry/Makefile new file mode 100644 index 0000000000..b13e99a354 --- /dev/null +++ b/src/test/modules/test_dsm_registry/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/test_dsm_registry/Makefile + +MODULE_big = test_dsm_registry +OBJS = \ + $(WIN32RES) \ + test_dsm_registry.o +PGFILEDESC = "test_dsm_registry - test code for the DSM registry" + +EXTENSION = test_dsm_registry +DATA = test_dsm_registry--1.0.sql + +REGRESS = test_dsm_registry + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_dsm_registry +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out new file mode 100644 index 0000000000..8ffbd343a0 --- /dev/null +++ b/src/test/modules/test_dsm_registry/expected/test_dsm_registry.out @@ -0,0 +1,14 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); + set_val_in_shmem +------------------ + +(1 row) + +\c +SELECT get_val_in_shmem(); + get_val_in_shmem +------------------ + 1236 +(1 row) + diff --git a/src/test/modules/test_dsm_registry/meson.build b/src/test/modules/test_dsm_registry/meson.build new file mode 100644 index 0000000000..4a7992109b --- /dev/null +++ b/src/test/modules/test_dsm_registry/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2023, PostgreSQL Global Development Group + +test_dsm_registry_sources = files( + 'test_dsm_registry.c', +) + +if host_system == 'windows' + test_dsm_registry_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_dsm_registry', + '--FILEDESC', 'test_dsm_registry - test code for the DSM registry',]) +endif + +test_dsm_registry = shared_module('test_dsm_registry', + test_dsm_registry_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_dsm_registry + +test_install_data += files( + 'test_dsm_registry.control', + 'test_dsm_registry--1.0.sql', +) + +tests += { + 'name': 'test_dsm_registry', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'regress': { + 'sql': [ + 'test_dsm_registry', + ], + }, +} diff --git a/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql new file mode 100644 index 0000000000..b3351be0a1 --- /dev/null +++ b/src/test/modules/test_dsm_registry/sql/test_dsm_registry.sql @@ -0,0 +1,4 @@ +CREATE EXTENSION test_dsm_registry; +SELECT set_val_in_shmem(1236); +\c +SELECT get_val_in_shmem(); diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql new file mode 100644 index 0000000000..8c55b0919b --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql @@ -0,0 +1,10 @@ +/* src/test/modules/test_dsm_registry/test_dsm_registry--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_dsm_registry" to load this file. \quit + +CREATE FUNCTION set_val_in_shmem(val INT) RETURNS VOID + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE FUNCTION get_val_in_shmem() RETURNS INT + AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.c b/src/test/modules/test_dsm_registry/test_dsm_registry.c new file mode 100644 index 0000000000..068f47ed02 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.c @@ -0,0 +1,75 @@ +/*-------------------------------------------------------------------------- + * + * test_dsm_registry.c + * Test the DSM registry + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_dsm_registry/test_dsm_registry.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "storage/dsm_registry.h" +#include "storage/lwlock.h" + +PG_MODULE_MAGIC; + +typedef struct TestDSMRegistryStruct +{ + int val; + LWLock lck; +} TestDSMRegistryStruct; + +static TestDSMRegistryStruct *tdr_state; + +static void +init_state(void *ptr) +{ + TestDSMRegistryStruct *state = (TestDSMRegistryStruct *) ptr; + + LWLockInitialize(&state->lck, LWLockNewTrancheId()); + state->val = 0; +} + +static void +dsm_registry_attach(void) +{ + bool found; + + tdr_state = dsm_registry_init_or_attach("test_dsm_registry", + sizeof(TestDSMRegistryStruct), + init_state, &found); + LWLockRegisterTranche(tdr_state->lck.tranche, "test_dsm_registry"); +} + +PG_FUNCTION_INFO_V1(set_val_in_shmem); +Datum +set_val_in_shmem(PG_FUNCTION_ARGS) +{ + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_EXCLUSIVE); + tdr_state->val = PG_GETARG_UINT32(0); + LWLockRelease(&tdr_state->lck); + + PG_RETURN_VOID(); +} + +PG_FUNCTION_INFO_V1(get_val_in_shmem); +Datum +get_val_in_shmem(PG_FUNCTION_ARGS) +{ + int ret; + + dsm_registry_attach(); + + LWLockAcquire(&tdr_state->lck, LW_SHARED); + ret = tdr_state->val; + LWLockRelease(&tdr_state->lck); + + PG_RETURN_UINT32(ret); +} diff --git a/src/test/modules/test_dsm_registry/test_dsm_registry.control b/src/test/modules/test_dsm_registry/test_dsm_registry.control new file mode 100644 index 0000000000..813f099889 --- /dev/null +++ b/src/test/modules/test_dsm_registry/test_dsm_registry.control @@ -0,0 +1,4 @@ +comment = 'Test code for the DSM registry' +default_version = '1.0' +module_pathname = '$libdir/test_dsm_registry' +relocatable = true diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index e37ef9aa76..4270109fb8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -610,6 +610,8 @@ DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt DropdbStmt +DSMRegistryCtxStruct +DSMRegistryEntry DumpComponents DumpId DumpOptions @@ -2796,6 +2798,7 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Time TempNamespaceStatus +TestDSMRegistryStruct TestDecodingData TestDecodingTxnData TestSpec -- 2.25.1 --Dxnq1zWXvFF0Q93v Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v3-0002-use-dsm-registry-for-pg_prewarm.patch" ^ permalink raw reply [nested|flat] 9+ messages in thread
end of thread, other threads:[~2023-10-12 03:07 UTC | newest] Thread overview: 9+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-12-22 06:23 [PATCH 03/20] Support multiple compression algs/levels/opts.. Justin Pryzby <pryzbyj@telsasoft.com> 2023-10-12 03:07 [PATCH v7 2/3] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v8 2/3] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v2 1/2] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v3 1/2] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v4 1/2] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v5 1/2] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v6 2/3] add dsm registry Nathan Bossart <nathan@postgresql.org> 2023-10-12 03:07 [PATCH v1 1/2] add dsm registry Nathan Bossart <nathan@postgresql.org>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox