agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH 09/10] Add zstd compression levels 4+ messages / 2 participants [nested] [flat]
* [PATCH 09/10] Add zstd compression levels @ 2021-03-14 22:12 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Justin Pryzby @ 2021-03-14 22:12 UTC (permalink / raw) --- src/backend/access/transam/xlog.c | 6 +++++- src/backend/access/transam/xloginsert.c | 15 +++++++++++++-- src/backend/access/transam/xlogreader.c | 12 ++++++++++++ src/backend/utils/misc/guc.c | 2 +- src/include/access/xlog_internal.h | 4 ++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 92023de9f5..b14c7c5929 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_ZSTD; +int wal_compression_method = WAL_COMPRESSION_ZSTD_FAST_10; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; @@ -192,6 +192,10 @@ const struct config_enum_entry wal_compression_options[] = { #endif #ifdef USE_ZSTD {"zstd", WAL_COMPRESSION_ZSTD, false}, + {"zstd-1", WAL_COMPRESSION_ZSTD_1, false}, + {"zstd-fast-10", WAL_COMPRESSION_ZSTD_FAST_10, false}, + {"zstd-fast-20", WAL_COMPRESSION_ZSTD_FAST_20, false}, + {"zstd-fast-40", WAL_COMPRESSION_ZSTD_FAST_40, false}, #endif {NULL, 0, false} }; diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 4591e476c6..4f11f96373 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -915,11 +915,22 @@ XLogCompressBackupBlock(char *page, uint16 hole_offset, uint16 hole_length, #ifdef USE_ZSTD case WAL_COMPRESSION_ZSTD: - len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, - ZSTD_CLEVEL_DEFAULT); + case WAL_COMPRESSION_ZSTD_1: + case WAL_COMPRESSION_ZSTD_FAST_10: + case WAL_COMPRESSION_ZSTD_FAST_20: + case WAL_COMPRESSION_ZSTD_FAST_40: + { + int level = compression == WAL_COMPRESSION_ZSTD_1 ? 1 : + compression == WAL_COMPRESSION_ZSTD_FAST_10 ? -10 : + compression == WAL_COMPRESSION_ZSTD_FAST_20 ? -20 : + compression == WAL_COMPRESSION_ZSTD_FAST_40 ? -40 : + ZSTD_CLEVEL_DEFAULT; + + len = ZSTD_compress(dest, COMPRESS_BUFSIZE, source, orig_len, level); if (ZSTD_isError(len)) len = -1; break; + } #endif default: diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index 0f9d522087..0de61d3073 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1554,6 +1554,10 @@ struct walcompression walmethods[] = { {"zlib", WAL_COMPRESSION_ZLIB}, {"lz4", WAL_COMPRESSION_LZ4}, {"zstd", WAL_COMPRESSION_ZSTD}, + {"zstd-1", WAL_COMPRESSION_ZSTD}, + {"zstd-fast-10",WAL_COMPRESSION_ZSTD}, + {"zstd-fast-20",WAL_COMPRESSION_ZSTD}, + {"zstd-fast-40",WAL_COMPRESSION_ZSTD}, }; /* @@ -1628,6 +1632,14 @@ RestoreBlockImage(XLogReaderState *record, uint8 block_id, char *page) #ifdef USE_ZSTD case WAL_COMPRESSION_ZSTD: + /* + * There aren't actually written into the header - decompression is the + * same. + * case WAL_COMPRESSION_ZSTD_1: + * case WAL_COMPRESSION_ZSTD_FAST_10: + * case WAL_COMPRESSION_ZSTD_FAST_20: + * case WAL_COMPRESSION_ZSTD_FAST_40: + */ decomp_result = ZSTD_decompress(tmp.data, BLCKSZ-bkpb->hole_length, ptr, bkpb->bimg_len); // XXX: ZSTD_getErrorName diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 8031e027aa..667fc4c0c1 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4728,7 +4728,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression_method, - WAL_COMPRESSION_ZSTD, wal_compression_options, + WAL_COMPRESSION_ZSTD_FAST_10, wal_compression_options, NULL, NULL, NULL }, diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index fa8146645d..a435b1a654 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -338,6 +338,10 @@ typedef enum WalCompression WAL_COMPRESSION_ZLIB, WAL_COMPRESSION_LZ4, WAL_COMPRESSION_ZSTD, + WAL_COMPRESSION_ZSTD_1, + WAL_COMPRESSION_ZSTD_FAST_10, /* level = -10 */ + WAL_COMPRESSION_ZSTD_FAST_20, /* level = -20 */ + WAL_COMPRESSION_ZSTD_FAST_40, /* level = -40 */ } WalCompression; extern const char *wal_compression_name(WalCompression compression); -- 2.17.0 --jozmn01XJZjDjM3N-- ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v4 4/4] cache sequence data @ 2024-07-18 03:13 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Nathan Bossart @ 2024-07-18 03:13 UTC (permalink / raw) --- src/bin/pg_dump/pg_dump.c | 91 +++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index a54e32c7be..14019907db 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -114,6 +114,8 @@ typedef struct int64 startv; /* start value */ int64 incby; /* increment value */ int64 cache; /* cache size */ + int64 last_value; /* last value of sequence */ + bool is_called; /* whether nextval advances before returning */ } SequenceItem; typedef enum OidOptions @@ -17237,16 +17239,30 @@ collectSequences(Archive *fout) * Before Postgres 10, sequence metadata is in the sequence itself. We * could likely make use of the sorted table with some extra effort, but * for now it seems unlikely to be worth it. + * + * Since version 18, we can gather the sequence data in this query with + * pg_sequence_read_tuple(), but we only do so for non-schema-only dumps. */ if (fout->remoteVersion < 100000) return; - - query = "SELECT seqrelid, format_type(seqtypid, NULL), " - "seqstart, seqincrement, " - "seqmax, seqmin, " - "seqcache, seqcycle " - "FROM pg_catalog.pg_sequence " - "ORDER BY seqrelid"; + else if (fout->remoteVersion < 180000 || + (fout->dopt->schemaOnly && !fout->dopt->sequence_data)) + query = "SELECT seqrelid, format_type(seqtypid, NULL), " + "seqstart, seqincrement, " + "seqmax, seqmin, " + "seqcache, seqcycle, " + "NULL, 'f' " + "FROM pg_catalog.pg_sequence " + "ORDER BY seqrelid"; + else + query = "SELECT seqrelid, format_type(seqtypid, NULL), " + "seqstart, seqincrement, " + "seqmax, seqmin, " + "seqcache, seqcycle, " + "last_value, is_called " + "FROM pg_catalog.pg_sequence, " + "pg_sequence_read_tuple(seqrelid) " + "ORDER BY seqrelid;"; res = ExecuteSqlQuery(fout, query, PGRES_TUPLES_OK); @@ -17269,6 +17285,8 @@ collectSequences(Archive *fout) sequences[i].minv = strtoi64(PQgetvalue(res, i, 5), NULL, 10); sequences[i].cache = strtoi64(PQgetvalue(res, i, 6), NULL, 10); sequences[i].cycled = (strcmp(PQgetvalue(res, i, 7), "t") == 0); + sequences[i].last_value = strtoi64(PQgetvalue(res, i, 8), NULL, 10); + sequences[i].is_called = (strcmp(PQgetvalue(res, i, 9), "t") == 0); } PQclear(res); @@ -17550,30 +17568,59 @@ static void dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) { TableInfo *tbinfo = tdinfo->tdtable; - PGresult *res; - char *last; + int64 last; bool called; PQExpBuffer query = createPQExpBuffer(); - appendPQExpBuffer(query, - "SELECT last_value, is_called FROM %s", - fmtQualifiedDumpable(tbinfo)); + /* + * For versions >= 18, the sequence information is gathered in the sorted + * array before any calls to dumpSequenceData(). See collectSequences() + * for more information. + * + * For older versions, we have to query the sequence relations + * individually. + */ + if (fout->remoteVersion < 180000) + { + PGresult *res; - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + appendPQExpBuffer(query, + "SELECT last_value, is_called FROM %s", + fmtQualifiedDumpable(tbinfo)); - if (PQntuples(res) != 1) - pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", - "query to get data of sequence \"%s\" returned %d rows (expected 1)", - PQntuples(res)), - tbinfo->dobj.name, PQntuples(res)); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - last = PQgetvalue(res, 0, 0); - called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + if (PQntuples(res) != 1) + pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", + "query to get data of sequence \"%s\" returned %d rows (expected 1)", + PQntuples(res)), + tbinfo->dobj.name, PQntuples(res)); + + last = strtoi64(PQgetvalue(res, 0, 0), NULL, 10); + called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + + PQclear(res); + } + else + { + SequenceItem key = {0}; + SequenceItem *entry; + + Assert(sequences); + Assert(tbinfo->dobj.catId.oid); + + key.oid = tbinfo->dobj.catId.oid; + entry = bsearch(&key, sequences, nsequences, + sizeof(SequenceItem), SequenceItemCmp); + + last = entry->last_value; + called = entry->is_called; + } resetPQExpBuffer(query); appendPQExpBufferStr(query, "SELECT pg_catalog.setval("); appendStringLiteralAH(query, fmtQualifiedDumpable(tbinfo), fout); - appendPQExpBuffer(query, ", %s, %s);\n", + appendPQExpBuffer(query, ", " INT64_FORMAT ", %s);\n", last, (called ? "true" : "false")); if (tdinfo->dobj.dump & DUMP_COMPONENT_DATA) @@ -17587,8 +17634,6 @@ dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) .deps = &(tbinfo->dobj.dumpId), .nDeps = 1)); - PQclear(res); - destroyPQExpBuffer(query); } -- 2.39.3 (Apple Git-146) --BezmxU4EbGlYNqqH-- ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v5 4/4] cache sequence data @ 2024-07-18 03:13 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Nathan Bossart @ 2024-07-18 03:13 UTC (permalink / raw) --- src/bin/pg_dump/pg_dump.c | 81 ++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 9d1707623b..2c5c614abd 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -131,6 +131,8 @@ typedef struct int64 startv; /* start value */ int64 incby; /* increment value */ int64 cache; /* cache size */ + int64 last_value; /* last value of sequence */ + bool is_called; /* whether nextval advances before returning */ } SequenceItem; typedef enum OidOptions @@ -17266,16 +17268,30 @@ collectSequences(Archive *fout) * Before Postgres 10, sequence metadata is in the sequence itself. We * could likely make use of the sorted table with some extra effort, but * for now it seems unlikely to be worth it. + * + * Since version 18, we can gather the sequence data in this query with + * pg_sequence_read_tuple(), but we only do so for non-schema-only dumps. */ if (fout->remoteVersion < 100000) return; - else + else if (fout->remoteVersion < 180000 || + (fout->dopt->schemaOnly && !fout->dopt->sequence_data)) query = "SELECT seqrelid, format_type(seqtypid, NULL), " "seqstart, seqincrement, " "seqmax, seqmin, " - "seqcache, seqcycle " + "seqcache, seqcycle, " + "NULL, 'f' " "FROM pg_catalog.pg_sequence " "ORDER BY seqrelid"; + else + query = "SELECT seqrelid, format_type(seqtypid, NULL), " + "seqstart, seqincrement, " + "seqmax, seqmin, " + "seqcache, seqcycle, " + "last_value, is_called " + "FROM pg_catalog.pg_sequence, " + "pg_sequence_read_tuple(seqrelid) " + "ORDER BY seqrelid;"; res = ExecuteSqlQuery(fout, query, PGRES_TUPLES_OK); @@ -17292,6 +17308,8 @@ collectSequences(Archive *fout) sequences[i].minv = strtoi64(PQgetvalue(res, i, 5), NULL, 10); sequences[i].cache = strtoi64(PQgetvalue(res, i, 6), NULL, 10); sequences[i].cycled = (strcmp(PQgetvalue(res, i, 7), "t") == 0); + sequences[i].last_value = strtoi64(PQgetvalue(res, i, 8), NULL, 10); + sequences[i].is_called = (strcmp(PQgetvalue(res, i, 9), "t") == 0); } PQclear(res); @@ -17558,30 +17576,59 @@ static void dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) { TableInfo *tbinfo = tdinfo->tdtable; - PGresult *res; - char *last; + int64 last; bool called; PQExpBuffer query = createPQExpBuffer(); - appendPQExpBuffer(query, - "SELECT last_value, is_called FROM %s", - fmtQualifiedDumpable(tbinfo)); + /* + * For versions >= 18, the sequence information is gathered in the sorted + * array before any calls to dumpSequenceData(). See collectSequences() + * for more information. + * + * For older versions, we have to query the sequence relations + * individually. + */ + if (fout->remoteVersion < 180000) + { + PGresult *res; - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + appendPQExpBuffer(query, + "SELECT last_value, is_called FROM %s", + fmtQualifiedDumpable(tbinfo)); - if (PQntuples(res) != 1) - pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", - "query to get data of sequence \"%s\" returned %d rows (expected 1)", - PQntuples(res)), - tbinfo->dobj.name, PQntuples(res)); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - last = PQgetvalue(res, 0, 0); - called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + if (PQntuples(res) != 1) + pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", + "query to get data of sequence \"%s\" returned %d rows (expected 1)", + PQntuples(res)), + tbinfo->dobj.name, PQntuples(res)); + + last = strtoi64(PQgetvalue(res, 0, 0), NULL, 10); + called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + + PQclear(res); + } + else + { + SequenceItem key = {0}; + SequenceItem *entry; + + Assert(sequences); + Assert(tbinfo->dobj.catId.oid); + + key.oid = tbinfo->dobj.catId.oid; + entry = bsearch(&key, sequences, nsequences, + sizeof(SequenceItem), SequenceItemCmp); + + last = entry->last_value; + called = entry->is_called; + } resetPQExpBuffer(query); appendPQExpBufferStr(query, "SELECT pg_catalog.setval("); appendStringLiteralAH(query, fmtQualifiedDumpable(tbinfo), fout); - appendPQExpBuffer(query, ", %s, %s);\n", + appendPQExpBuffer(query, ", " INT64_FORMAT ", %s);\n", last, (called ? "true" : "false")); if (tdinfo->dobj.dump & DUMP_COMPONENT_DATA) @@ -17595,8 +17642,6 @@ dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) .deps = &(tbinfo->dobj.dumpId), .nDeps = 1)); - PQclear(res); - destroyPQExpBuffer(query); } -- 2.39.3 (Apple Git-146) --6k61W1XP/ccyRu6t-- ^ permalink raw reply [nested|flat] 4+ messages in thread
* [PATCH v6 4/4] cache sequence data @ 2024-07-18 03:13 Nathan Bossart <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Nathan Bossart @ 2024-07-18 03:13 UTC (permalink / raw) --- src/bin/pg_dump/pg_dump.c | 81 ++++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 8c42fd762d..78f6f608bc 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -131,6 +131,8 @@ typedef struct int64 startv; /* start value */ int64 incby; /* increment value */ int64 cache; /* cache size */ + int64 last_value; /* last value of sequence */ + bool is_called; /* whether nextval advances before returning */ } SequenceItem; typedef enum OidOptions @@ -17267,16 +17269,30 @@ collectSequences(Archive *fout) * Before Postgres 10, sequence metadata is in the sequence itself. We * could likely make use of the sorted table with some extra effort, but * for now it seems unlikely to be worth it. + * + * Since version 18, we can gather the sequence data in this query with + * pg_sequence_read_tuple(), but we only do so for non-schema-only dumps. */ if (fout->remoteVersion < 100000) return; - else + else if (fout->remoteVersion < 180000 || + (fout->dopt->schemaOnly && !fout->dopt->sequence_data)) query = "SELECT seqrelid, format_type(seqtypid, NULL), " "seqstart, seqincrement, " "seqmax, seqmin, " - "seqcache, seqcycle " + "seqcache, seqcycle, " + "NULL, 'f' " "FROM pg_catalog.pg_sequence " "ORDER BY seqrelid"; + else + query = "SELECT seqrelid, format_type(seqtypid, NULL), " + "seqstart, seqincrement, " + "seqmax, seqmin, " + "seqcache, seqcycle, " + "last_value, is_called " + "FROM pg_catalog.pg_sequence, " + "pg_sequence_read_tuple(seqrelid) " + "ORDER BY seqrelid;"; res = ExecuteSqlQuery(fout, query, PGRES_TUPLES_OK); @@ -17293,6 +17309,8 @@ collectSequences(Archive *fout) sequences[i].minv = strtoi64(PQgetvalue(res, i, 5), NULL, 10); sequences[i].cache = strtoi64(PQgetvalue(res, i, 6), NULL, 10); sequences[i].cycled = (strcmp(PQgetvalue(res, i, 7), "t") == 0); + sequences[i].last_value = strtoi64(PQgetvalue(res, i, 8), NULL, 10); + sequences[i].is_called = (strcmp(PQgetvalue(res, i, 9), "t") == 0); } PQclear(res); @@ -17559,30 +17577,59 @@ static void dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) { TableInfo *tbinfo = tdinfo->tdtable; - PGresult *res; - char *last; + int64 last; bool called; PQExpBuffer query = createPQExpBuffer(); - appendPQExpBuffer(query, - "SELECT last_value, is_called FROM %s", - fmtQualifiedDumpable(tbinfo)); + /* + * For versions >= 18, the sequence information is gathered in the sorted + * array before any calls to dumpSequenceData(). See collectSequences() + * for more information. + * + * For older versions, we have to query the sequence relations + * individually. + */ + if (fout->remoteVersion < 180000) + { + PGresult *res; - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + appendPQExpBuffer(query, + "SELECT last_value, is_called FROM %s", + fmtQualifiedDumpable(tbinfo)); - if (PQntuples(res) != 1) - pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", - "query to get data of sequence \"%s\" returned %d rows (expected 1)", - PQntuples(res)), - tbinfo->dobj.name, PQntuples(res)); + res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); - last = PQgetvalue(res, 0, 0); - called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + if (PQntuples(res) != 1) + pg_fatal(ngettext("query to get data of sequence \"%s\" returned %d row (expected 1)", + "query to get data of sequence \"%s\" returned %d rows (expected 1)", + PQntuples(res)), + tbinfo->dobj.name, PQntuples(res)); + + last = strtoi64(PQgetvalue(res, 0, 0), NULL, 10); + called = (strcmp(PQgetvalue(res, 0, 1), "t") == 0); + + PQclear(res); + } + else + { + SequenceItem key = {0}; + SequenceItem *entry; + + Assert(sequences); + Assert(tbinfo->dobj.catId.oid); + + key.oid = tbinfo->dobj.catId.oid; + entry = bsearch(&key, sequences, nsequences, + sizeof(SequenceItem), SequenceItemCmp); + + last = entry->last_value; + called = entry->is_called; + } resetPQExpBuffer(query); appendPQExpBufferStr(query, "SELECT pg_catalog.setval("); appendStringLiteralAH(query, fmtQualifiedDumpable(tbinfo), fout); - appendPQExpBuffer(query, ", %s, %s);\n", + appendPQExpBuffer(query, ", " INT64_FORMAT ", %s);\n", last, (called ? "true" : "false")); if (tdinfo->dobj.dump & DUMP_COMPONENT_DATA) @@ -17596,8 +17643,6 @@ dumpSequenceData(Archive *fout, const TableDataInfo *tdinfo) .deps = &(tbinfo->dobj.dumpId), .nDeps = 1)); - PQclear(res); - destroyPQExpBuffer(query); } -- 2.39.3 (Apple Git-146) --zrchpDNjnCAXEpsr-- ^ permalink raw reply [nested|flat] 4+ messages in thread
end of thread, other threads:[~2024-07-18 03:13 UTC | newest] Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-14 22:12 [PATCH 09/10] Add zstd compression levels Justin Pryzby <[email protected]> 2024-07-18 03:13 [PATCH v4 4/4] cache sequence data Nathan Bossart <[email protected]> 2024-07-18 03:13 [PATCH v5 4/4] cache sequence data Nathan Bossart <[email protected]> 2024-07-18 03:13 [PATCH v6 4/4] cache sequence data Nathan Bossart <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox