public inbox for [email protected]
help / color / mirror / Atom feedFrom: Boris Mironov <[email protected]>
To: Madyshev Egor <[email protected]>
To: [email protected] <[email protected]>
Subject: Re: Idea to enhance pgbench by more modes to generate data (multi-TXNs, UNNEST, COPY BINARY)
Date: Tue, 3 Mar 2026 11:13:01 +0000
Message-ID: <PH0PR08MB702059923096E4BAA78FED02887FA@PH0PR08MB7020.namprd08.prod.outlook.com> (raw)
In-Reply-To: <PH0PR08MB70202953137489C2DE455005887FA@PH0PR08MB7020.namprd08.prod.outlook.com>
References: <DS0PR08MB9565D91414C65B3AC363825488CFA@DS0PR08MB9565.namprd08.prod.outlook.com>
<CAExHW5vtdtd5QnFobxOhbVeh5jk3_61zYRxLqCXEZpO3jOeFDg@mail.gmail.com>
<DS0PR08MB9565C0B8B45F7B8D78D89EA188CAA@DS0PR08MB9565.namprd08.prod.outlook.com>
<CAExHW5umH2K5w_5GRCWGrRHfceMjVn72hF=z_A7G+vReryx0uw@mail.gmail.com>
<DS0PR08MB956560D79EA051E98688F78088CAA@DS0PR08MB9565.namprd08.prod.outlook.com>
<CAExHW5vFMkBfv9zB3c5gxF=VLkR7dxM2a3pSFsNuz4JYqc8wRA@mail.gmail.com>
<PH0PR08MB7020CE2AE1B6937BD01B852F88C9A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB702059D610C7D84594CD3BB388D5A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB70201EC778A20A2BDC8A5F4A88D2A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB7020802F14B1AC122AAB6F7588D3A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB702075EB12438C2F9A32CD26889EA@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB70205C9C827602026F60FA07889FA@PH0PR08MB7020.namprd08.prod.outlook.com>
<[email protected]>
<PH0PR08MB70206B42F5A79A518B610815889FA@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB70204888D724794BA83EA6E58866A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB70207B96C493E5777DB4D6428866A@PH0PR08MB7020.namprd08.prod.outlook.com>
<[email protected]>
<PH0PR08MB70208182284EBA18F64228798877A@PH0PR08MB7020.namprd08.prod.outlook.com>
<PH0PR08MB702074674145021B680D9D6C8871A@PH0PR08MB7020.namprd08.prod.outlook.com>
<[email protected]>
<PH0PR08MB70202953137489C2DE455005887FA@PH0PR08MB7020.namprd08.prod.outlook.com>
Hi Egor,
Sorry for so many extra changes. I promise, this one is good.
I believe it will beneficial to explain how buffer size for each row in COPY BINARY is actually calculated. Instead of hard coded values like 35, 40, 250, I put in comments actual sizes of each field as well as easy to read formulas. IMHO it adds extra value and resolves possible future questions right away.
I greatly appreciate your time spent on this patch.
Best regards,
Boris
Attachments:
[application/octet-stream] v11-pgbench-faster-modes.patch (133.2K, 3-v11-pgbench-faster-modes.patch)
download | inline diff:
From c768f399c556295de7d53895410e686d86b4b960 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 9 Nov 2025 19:34:58 +0700
Subject: [PATCH v11 01/28] Converting one huge transaction into series of one
per 'scale'
---
src/bin/pgbench/pgbench.c | 61 ++++++++++++++++++++++++++-------------
1 file changed, 41 insertions(+), 20 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index d8764ba6fe0..284a7c860f1 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -181,6 +181,12 @@ static int64 end_time = 0; /* when to stop in micro seconds, under -T */
*/
static int scale = 1;
+/*
+ * scaling factor after which we switch to multiple transactions during
+ * data population phase on server side
+ */
+static int64 single_txn_scale_limit = 1;
+
/*
* fillfactor. for example, fillfactor = 90 will use only 90 percent
* space during inserts and leave 10 percent free.
@@ -5213,6 +5219,7 @@ static void
initGenerateDataServerSide(PGconn *con)
{
PQExpBufferData sql;
+ int chunk = (scale >= single_txn_scale_limit) ? 1 : scale;
fprintf(stderr, "generating data (server-side)...\n");
@@ -5225,30 +5232,44 @@ initGenerateDataServerSide(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
+ executeStatement(con, "commit");
+
initPQExpBuffer(&sql);
- printfPQExpBuffer(&sql,
- "insert into pgbench_branches(bid,bbalance) "
- "select bid, 0 "
- "from generate_series(1, %d) as bid", nbranches * scale);
- executeStatement(con, sql.data);
-
- printfPQExpBuffer(&sql,
- "insert into pgbench_tellers(tid,bid,tbalance) "
- "select tid, (tid - 1) / %d + 1, 0 "
- "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
- executeStatement(con, sql.data);
-
- printfPQExpBuffer(&sql,
- "insert into pgbench_accounts(aid,bid,abalance,filler) "
- "select aid, (aid - 1) / %d + 1, 0, '' "
- "from generate_series(1, " INT64_FORMAT ") as aid",
- naccounts, (int64) naccounts * scale);
- executeStatement(con, sql.data);
+ for (int i = 0; i < scale; i += chunk) {
+ executeStatement(con, "begin");
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_branches(bid,bbalance) "
+ "select bid + 1, 0 "
+ "from generate_series(%d, %d) as bid", i, i + chunk);
+ //"select bid, 0 "
+ //"from generate_series(1, %d) as bid", nbranches * scale);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_tellers(tid,bid,tbalance) "
+ "select tid + 1, tid / %d + 1, 0 "
+ "from generate_series(%d, %d) as tid",
+ ntellers, i * ntellers, (i + chunk) * ntellers - 1);
+ //"select tid, (tid - 1) / %d + 1, 0 "
+ //"from generate_series(1, %d) as tid", ntellers, ntellers * scale);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_accounts(aid,bid,abalance,filler) "
+ "select aid + 1, aid / %d + 1, 0, '' "
+ "from generate_series(" INT64_FORMAT ", " INT64_FORMAT ") as aid",
+ naccounts, (int64) i * naccounts, (int64) (i + chunk) * naccounts - 1);
+ //"select aid, (aid - 1) / %d + 1, 0, '' "
+ //"from generate_series(1, " INT64_FORMAT ") as aid",
+ //naccounts, (int64) naccounts * scale);
+ executeStatement(con, sql.data);
+
+ executeStatement(con, "commit");
+ }
termPQExpBuffer(&sql);
-
- executeStatement(con, "commit");
}
/*
--
2.43.0
From 0eddb156c187d829c4381bc928c5314705928852 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 9 Nov 2025 20:13:23 +0700
Subject: [PATCH v11 02/28] Getting rid off limit for single transaction size
during data generation
---
src/bin/pgbench/pgbench.c | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 284a7c860f1..28b72e4cf1f 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -181,12 +181,6 @@ static int64 end_time = 0; /* when to stop in micro seconds, under -T */
*/
static int scale = 1;
-/*
- * scaling factor after which we switch to multiple transactions during
- * data population phase on server side
- */
-static int64 single_txn_scale_limit = 1;
-
/*
* fillfactor. for example, fillfactor = 90 will use only 90 percent
* space during inserts and leave 10 percent free.
@@ -5219,7 +5213,6 @@ static void
initGenerateDataServerSide(PGconn *con)
{
PQExpBufferData sql;
- int chunk = (scale >= single_txn_scale_limit) ? 1 : scale;
fprintf(stderr, "generating data (server-side)...\n");
@@ -5236,13 +5229,13 @@ initGenerateDataServerSide(PGconn *con)
initPQExpBuffer(&sql);
- for (int i = 0; i < scale; i += chunk) {
+ for (int i = 0; i < scale; i++) {
executeStatement(con, "begin");
printfPQExpBuffer(&sql,
"insert into pgbench_branches(bid,bbalance) "
"select bid + 1, 0 "
- "from generate_series(%d, %d) as bid", i, i + chunk);
+ "from generate_series(%d, %d) as bid", i, i + 1);
//"select bid, 0 "
//"from generate_series(1, %d) as bid", nbranches * scale);
executeStatement(con, sql.data);
@@ -5251,7 +5244,7 @@ initGenerateDataServerSide(PGconn *con)
"insert into pgbench_tellers(tid,bid,tbalance) "
"select tid + 1, tid / %d + 1, 0 "
"from generate_series(%d, %d) as tid",
- ntellers, i * ntellers, (i + chunk) * ntellers - 1);
+ ntellers, i * ntellers, (i + 1) * ntellers - 1);
//"select tid, (tid - 1) / %d + 1, 0 "
//"from generate_series(1, %d) as tid", ntellers, ntellers * scale);
executeStatement(con, sql.data);
@@ -5260,7 +5253,7 @@ initGenerateDataServerSide(PGconn *con)
"insert into pgbench_accounts(aid,bid,abalance,filler) "
"select aid + 1, aid / %d + 1, 0, '' "
"from generate_series(" INT64_FORMAT ", " INT64_FORMAT ") as aid",
- naccounts, (int64) i * naccounts, (int64) (i + chunk) * naccounts - 1);
+ naccounts, (int64) i * naccounts, (int64) (i + 1) * naccounts - 1);
//"select aid, (aid - 1) / %d + 1, 0, '' "
//"from generate_series(1, " INT64_FORMAT ") as aid",
//naccounts, (int64) naccounts * scale);
--
2.43.0
From c5659cf474ec273c057668f30a4f435fd02f2da7 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 9 Nov 2025 20:38:36 +0700
Subject: [PATCH v11 03/28] No need to keep old code in comments
---
src/bin/pgbench/pgbench.c | 7 -------
1 file changed, 7 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 28b72e4cf1f..97895aa9edf 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5236,8 +5236,6 @@ initGenerateDataServerSide(PGconn *con)
"insert into pgbench_branches(bid,bbalance) "
"select bid + 1, 0 "
"from generate_series(%d, %d) as bid", i, i + 1);
- //"select bid, 0 "
- //"from generate_series(1, %d) as bid", nbranches * scale);
executeStatement(con, sql.data);
printfPQExpBuffer(&sql,
@@ -5245,8 +5243,6 @@ initGenerateDataServerSide(PGconn *con)
"select tid + 1, tid / %d + 1, 0 "
"from generate_series(%d, %d) as tid",
ntellers, i * ntellers, (i + 1) * ntellers - 1);
- //"select tid, (tid - 1) / %d + 1, 0 "
- //"from generate_series(1, %d) as tid", ntellers, ntellers * scale);
executeStatement(con, sql.data);
printfPQExpBuffer(&sql,
@@ -5254,9 +5250,6 @@ initGenerateDataServerSide(PGconn *con)
"select aid + 1, aid / %d + 1, 0, '' "
"from generate_series(" INT64_FORMAT ", " INT64_FORMAT ") as aid",
naccounts, (int64) i * naccounts, (int64) (i + 1) * naccounts - 1);
- //"select aid, (aid - 1) / %d + 1, 0, '' "
- //"from generate_series(1, " INT64_FORMAT ") as aid",
- //naccounts, (int64) naccounts * scale);
executeStatement(con, sql.data);
executeStatement(con, "commit");
--
2.43.0
From e47b52ddf23593dad9375ef5356fd41d0621ede3 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 10 Nov 2025 19:06:48 +0700
Subject: [PATCH v11 04/28] Adding server-side data generation via unnest
---
src/bin/pgbench/pgbench.c | 199 ++++++++++++++++++++++++++++++++++----
1 file changed, 182 insertions(+), 17 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 97895aa9edf..65d77cdefea 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -161,7 +161,7 @@ typedef struct socket_set
* some configurable parameters */
#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
-#define ALL_INIT_STEPS "dtgGvpf" /* all possible steps */
+#define ALL_INIT_STEPS "dtgGiIvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
#define DEFAULT_NXACTS 10 /* default nxacts */
@@ -171,6 +171,12 @@ typedef struct socket_set
#define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
#define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
+/* original single transaction server-side method */
+#define GEN_TYPE_INSERT_ORIGINAL 'G' /* use INSERT .. SELECT generate_series to generate data */
+/* 'one transaction per scale' server-side methods */
+#define GEN_TYPE_INSERT_SERIES 'i' /* use INSERT .. SELECT generate_series to generate data */
+#define GEN_TYPE_INSERT_UNNEST 'I' /* use INSERT .. SELECT unnest to generate data */
+
static int nxacts = 0; /* number of transactions per client */
static int duration = 0; /* duration in seconds */
static int64 end_time = 0; /* when to stop in micro seconds, under -T */
@@ -181,6 +187,11 @@ static int64 end_time = 0; /* when to stop in micro seconds, under -T */
*/
static int scale = 1;
+/*
+ *
+ */
+static char data_generation_type = '?';
+
/*
* fillfactor. for example, fillfactor = 90 will use only 90 percent
* space during inserts and leave 10 percent free.
@@ -914,7 +925,9 @@ usage(void)
" d: drop any existing pgbench tables\n"
" t: create the tables used by the standard pgbench scenario\n"
" g: generate data, client-side\n"
- " G: generate data, server-side\n"
+ " G: generate data, server-side in single transaction\n"
+ " i: server-side (multiple TXNs) INSERT .. SELECT generate_series\n"
+ " I: server-side (multiple TXNs) INSERT .. SELECT unnest\n"
" v: invoke VACUUM on the standard tables\n"
" p: create primary key indexes on the standard tables\n"
" f: create foreign keys between the standard tables\n"
@@ -5203,18 +5216,16 @@ initGenerateDataClientSide(PGconn *con)
}
/*
- * Fill the standard tables with some data generated on the server
- *
- * As already the case with the client-side data generation, the filler
- * column defaults to NULL in pgbench_branches and pgbench_tellers,
- * and is a blank-padded string in pgbench_accounts.
+ * Generating data via INSERT .. SELECT .. FROM generate_series
+ * whole dataset in single transaction
*/
static void
-initGenerateDataServerSide(PGconn *con)
+generateDataInsertSingleTXN(PGconn *con)
{
PQExpBufferData sql;
- fprintf(stderr, "generating data (server-side)...\n");
+ fprintf(stderr, "via INSERT .. SELECT generate_series... in single TXN\n");
+
/*
* we do all of this in one transaction to enable the backend's
@@ -5225,31 +5236,136 @@ initGenerateDataServerSide(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
+ initPQExpBuffer(&sql);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_branches(bid, bbalance) "
+ "select bid, 0 "
+ "from generate_series(1, %d)", scale * nbranches);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_tellers(tid, bid, tbalance) "
+ "select tid + 1, tid / %d + 1, 0 "
+ "from generate_series(0, %d) as tid",
+ ntellers, (scale * ntellers) - 1);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_accounts(aid, bid, abalance, "
+ "filler) "
+ "select aid + 1, aid / %d + 1, 0, '' "
+ "from generate_series(0, " INT64_FORMAT ") as aid",
+ naccounts, (int64) (scale * naccounts) - 1);
+ executeStatement(con, sql.data);
+
executeStatement(con, "commit");
+ termPQExpBuffer(&sql);
+}
+
+
+/*
+ * Generating data via INSERT .. SELECT .. FROM generate_series
+ * One transaction per 'scale'
+ */
+static void
+generateDataInsertSeries(PGconn *con)
+{
+ PQExpBufferData sql;
+
+ fprintf(stderr, "via INSERT .. SELECT generate_series... in multiple TXN(s)\n");
+
initPQExpBuffer(&sql);
- for (int i = 0; i < scale; i++) {
+ executeStatement(con, "begin");
+
+ /* truncate away any old data */
+ initTruncateTables(con);
+
+ executeStatement(con, "commit");
+
+ for (int i = 0; i < scale; i++)
+ {
executeStatement(con, "begin");
printfPQExpBuffer(&sql,
- "insert into pgbench_branches(bid,bbalance) "
- "select bid + 1, 0 "
- "from generate_series(%d, %d) as bid", i, i + 1);
+ "insert into pgbench_branches(bid, bbalance) "
+ "values(%d, 0)", i + 1);
executeStatement(con, sql.data);
printfPQExpBuffer(&sql,
- "insert into pgbench_tellers(tid,bid,tbalance) "
+ "insert into pgbench_tellers(tid, bid, tbalance) "
"select tid + 1, tid / %d + 1, 0 "
"from generate_series(%d, %d) as tid",
ntellers, i * ntellers, (i + 1) * ntellers - 1);
executeStatement(con, sql.data);
printfPQExpBuffer(&sql,
- "insert into pgbench_accounts(aid,bid,abalance,filler) "
+ "insert into pgbench_accounts(aid, bid, abalance, "
+ "filler) "
"select aid + 1, aid / %d + 1, 0, '' "
- "from generate_series(" INT64_FORMAT ", " INT64_FORMAT ") as aid",
- naccounts, (int64) i * naccounts, (int64) (i + 1) * naccounts - 1);
+ "from generate_series(" INT64_FORMAT ", "
+ INT64_FORMAT ") as aid",
+ naccounts, (int64) i * naccounts,
+ (int64) (i + 1) * naccounts - 1);
+ executeStatement(con, sql.data);
+
+ executeStatement(con, "commit");
+ }
+
+ termPQExpBuffer(&sql);
+}
+
+/*
+ * Generating data via INSERT .. SELECT .. FROM unnest
+ * One transaction per 'scale'
+ */
+static void
+generateDataInsertUnnest(PGconn *con)
+{
+ PQExpBufferData sql;
+
+ fprintf(stderr, "via INSERT .. SELECT unnest...\n");
+
+ initPQExpBuffer(&sql);
+
+ executeStatement(con, "begin");
+
+ /* truncate away any old data */
+ initTruncateTables(con);
+
+ executeStatement(con, "commit");
+
+ for (int s = 0; s < scale; s++)
+ {
+ executeStatement(con, "begin");
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_branches(bid,bbalance) "
+ "values(%d, 0)", s + 1);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "insert into pgbench_tellers(tid, bid, tbalance) "
+ "select unnest(array_agg(s.i order by s.i)) as tid, "
+ "%d as bid, 0 as tbalance "
+ "from generate_series(%d, %d) as s(i)",
+ s + 1, s * ntellers + 1, (s + 1) * ntellers);
+ executeStatement(con, sql.data);
+
+ printfPQExpBuffer(&sql,
+ "with data as ("
+ " select generate_series(" INT64_FORMAT ", "
+ INT64_FORMAT ") as i) "
+ "insert into pgbench_accounts(aid, bid, "
+ "abalance, filler) "
+ "select unnest(aid), unnest(bid), 0 as abalance, "
+ "'' as filler "
+ "from (select array_agg(i+1) aid, "
+ "array_agg(i/%d + 1) bid from data)",
+ (int64) s * naccounts + 1,
+ (int64) (s + 1) * naccounts, naccounts);
executeStatement(con, sql.data);
executeStatement(con, "commit");
@@ -5258,6 +5374,32 @@ initGenerateDataServerSide(PGconn *con)
termPQExpBuffer(&sql);
}
+/*
+ * Fill the standard tables with some data generated on the server
+ *
+ * As already the case with the client-side data generation, the filler
+ * column defaults to NULL in pgbench_branches and pgbench_tellers,
+ * and is a blank-padded string in pgbench_accounts.
+ */
+static void
+initGenerateDataServerSide(PGconn *con)
+{
+ fprintf(stderr, "generating data (server-side) ");
+
+ switch (data_generation_type)
+ {
+ case GEN_TYPE_INSERT_ORIGINAL:
+ generateDataInsertSingleTXN(con);
+ break;
+ case GEN_TYPE_INSERT_SERIES:
+ generateDataInsertSeries(con);
+ break;
+ case GEN_TYPE_INSERT_UNNEST:
+ generateDataInsertUnnest(con);
+ break;
+ }
+}
+
/*
* Invoke vacuum on the standard tables
*/
@@ -5341,6 +5483,8 @@ initCreateFKeys(PGconn *con)
static void
checkInitSteps(const char *initialize_steps)
{
+ char data_init_type = 0;
+
if (initialize_steps[0] == '\0')
pg_fatal("no initialization steps specified");
@@ -5352,7 +5496,26 @@ checkInitSteps(const char *initialize_steps)
pg_log_error_detail("Allowed step characters are: \"" ALL_INIT_STEPS "\".");
exit(1);
}
+
+ switch (*step)
+ {
+ case 'G':
+ data_init_type++;
+ data_generation_type = *step;
+ break;
+ case 'i':
+ data_init_type++;
+ data_generation_type = *step;
+ break;
+ case 'I':
+ data_init_type++;
+ data_generation_type = *step;
+ break;
+ }
}
+
+ if (data_init_type > 1)
+ pg_log_error("WARNING! More than one type of server-side data generation is requested");
}
/*
@@ -5395,6 +5558,8 @@ runInitSteps(const char *initialize_steps)
initGenerateDataClientSide(con);
break;
case 'G':
+ case 'i':
+ case 'I':
op = "server-side generate";
initGenerateDataServerSide(con);
break;
--
2.43.0
From 5e1827b889b283f50299ce6ab1a73f9f55a4a84f Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 10 Nov 2025 20:00:56 +0700
Subject: [PATCH v11 05/28] Fixing typo in query
---
src/bin/pgbench/pgbench.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 65d77cdefea..03e37df4434 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5241,7 +5241,8 @@ generateDataInsertSingleTXN(PGconn *con)
printfPQExpBuffer(&sql,
"insert into pgbench_branches(bid, bbalance) "
"select bid, 0 "
- "from generate_series(1, %d)", scale * nbranches);
+ "from generate_series(1, %d) as bid",
+ scale * nbranches);
executeStatement(con, sql.data);
printfPQExpBuffer(&sql,
--
2.43.0
From 7ca86521fda6929b8e0de3fc77dcbb8984009c88 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Tue, 11 Nov 2025 19:39:45 +0700
Subject: [PATCH v11 06/28] Adding support for COPY BINARY mode
---
src/bin/pgbench/pgbench.c | 393 ++++++++++++++++++++++++++++++++++++--
1 file changed, 381 insertions(+), 12 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 03e37df4434..71aa1d9479f 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -161,7 +161,7 @@ typedef struct socket_set
* some configurable parameters */
#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
-#define ALL_INIT_STEPS "dtgGiIvpf" /* all possible steps */
+#define ALL_INIT_STEPS "dtgCGiIvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
#define DEFAULT_NXACTS 10 /* default nxacts */
@@ -176,6 +176,8 @@ typedef struct socket_set
/* 'one transaction per scale' server-side methods */
#define GEN_TYPE_INSERT_SERIES 'i' /* use INSERT .. SELECT generate_series to generate data */
#define GEN_TYPE_INSERT_UNNEST 'I' /* use INSERT .. SELECT unnest to generate data */
+#define GEN_TYPE_COPY_ORIGINAL 'g' /* use COPY .. FROM STDIN .. TEXT to generate data */
+#define GEN_TYPE_COPY_BINARY 'C' /* use COPY .. FROM STDIN .. BINARY to generate data */
static int nxacts = 0; /* number of transactions per client */
static int duration = 0; /* duration in seconds */
@@ -188,10 +190,17 @@ static int64 end_time = 0; /* when to stop in micro seconds, under -T */
static int scale = 1;
/*
- *
+ * mode of data generation to use
*/
static char data_generation_type = '?';
+/*
+ * COPY FROM BINARY execution buffer
+ */
+#define BIN_COPY_BUF_SIZE 102400 /* maximum buffer size for COPY FROM BINARY */
+static char *bin_copy_buffer = NULL; /* buffer for COPY FROM BINARY */
+static int32_t bin_copy_buffer_length = 0; /* current buffer size */
+
/*
* fillfactor. for example, fillfactor = 90 will use only 90 percent
* space during inserts and leave 10 percent free.
@@ -861,7 +870,8 @@ static int wait_on_socket_set(socket_set *sa, int64 usecs);
static bool socket_has_input(socket_set *sa, int fd, int idx);
/* callback used to build rows for COPY during data loading */
-typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
+typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
+typedef void (*initRowMethodBin) (PGconn *con, PGresult *res, int64_t curr, int32_t parent);
/* callback functions for our flex lexer */
static const PsqlScanCallbacks pgbench_callbacks = {
@@ -925,6 +935,7 @@ usage(void)
" d: drop any existing pgbench tables\n"
" t: create the tables used by the standard pgbench scenario\n"
" g: generate data, client-side\n"
+ " C: client-side (single TNX) COPY .. FROM STDIN .. BINARY\n"
" G: generate data, server-side in single transaction\n"
" i: server-side (multiple TXNs) INSERT .. SELECT generate_series\n"
" I: server-side (multiple TXNs) INSERT .. SELECT unnest\n"
@@ -5191,9 +5202,9 @@ initPopulateTable(PGconn *con, const char *table, int64 base,
* a blank-padded string in pgbench_accounts.
*/
static void
-initGenerateDataClientSide(PGconn *con)
+initGenerateDataClientSideText(PGconn *con)
{
- fprintf(stderr, "generating data (client-side)...\n");
+ fprintf(stderr, "TEXT mode...\n");
/*
* we do all of this in one transaction to enable the backend's
@@ -5209,12 +5220,373 @@ initGenerateDataClientSide(PGconn *con)
* already exist
*/
initPopulateTable(con, "pgbench_branches", nbranches, initBranch);
- initPopulateTable(con, "pgbench_tellers", ntellers, initTeller);
+ initPopulateTable(con, "pgbench_tellers", ntellers, initTeller);
initPopulateTable(con, "pgbench_accounts", naccounts, initAccount);
executeStatement(con, "commit");
}
+
+/*
+ * Dumps binary buffer to file (purely for debugging)
+ */
+static void
+dumpBufferToFile(char *filename)
+{
+ FILE *file_ptr;
+ size_t bytes_written;
+
+ file_ptr = fopen(filename, "wb");
+ if (file_ptr == NULL)
+ {
+ fprintf(stderr, "Error opening file %s\n", filename);
+ return; // EXIT_FAILURE;
+ }
+
+ bytes_written = fwrite(bin_copy_buffer, 1, bin_copy_buffer_length, file_ptr);
+
+ if (bytes_written != bin_copy_buffer_length)
+ {
+ fprintf(stderr, "Error writing to file or incomplete write\n");
+ fclose(file_ptr);
+ return; // EXIT_FAILURE;
+ }
+
+ fclose(file_ptr);
+}
+
+/*
+ * Save char data to buffer
+ */
+static void
+bufferCharData(char *src, int32_t len)
+{
+ memcpy((char *) bin_copy_buffer + bin_copy_buffer_length, (char *) src, len);
+ bin_copy_buffer_length += len;
+}
+
+/*
+ * Converts platform byte order into network byte order
+ * SPARC doesn't reqire that
+ */
+static void
+bufferData(void *src, int32_t len)
+{
+#ifdef __sparc__
+ bufferCharData(src, len);
+#else
+ if (len == 1)
+ bufferCharData(src, len);
+ else
+ for (int32_t i = 0; i < len; i++)
+ {
+ ((char *) bin_copy_buffer + bin_copy_buffer_length)[i] =
+ ((char *) src)[len - i - 1];
+ }
+
+ bin_copy_buffer_length += len;
+#endif
+}
+
+/*
+ * adds column counter
+ */
+static void
+addColumnCounter(int16_t n)
+{
+ bufferData((void *) &n, sizeof(n));
+}
+
+/*
+ * adds column with NULL value
+ */
+static void
+addNullColumn()
+{
+ int32_t null = -1;
+ bufferData((void *) &null, sizeof(null));
+}
+
+/*
+ * adds column with int8 value
+ */
+static void
+addInt8Column(int8_t value)
+{
+ int8_t data = value;
+ int32_t size = sizeof(data);
+ bufferData((void *) &size, sizeof(size));
+ bufferData((void *) &data, sizeof(data));
+}
+
+/*
+ * adds column with int16 value
+ */
+static void
+addInt16Column(int16_t value)
+{
+ int16_t data = value;
+ int32_t size = sizeof(data);
+ bufferData((void *) &size, sizeof(size));
+ bufferData((void *) &data, sizeof(data));
+}
+
+/*
+ * adds column with inti32 value
+ */
+static void
+addInt32Column(int32_t value)
+{
+ int32_t data = value;
+ int32_t size = sizeof(data);
+ bufferData((void *) &size, sizeof(size));
+ bufferData((void *) &data, sizeof(data));
+}
+
+/*
+ * adds column with inti64 value
+ */
+static void
+addInt64Column(int64_t value)
+{
+ int64_t data = value;
+ int32_t size = sizeof(data);
+ bufferData((void *) &size, sizeof(size));
+ bufferData((void *) &data, sizeof(data));
+}
+
+/*
+ * adds column with char value
+ */
+static void
+addCharColumn(char *value)
+{
+ int32_t size = strlen(value);
+ bufferData((void *) &size, sizeof(size));
+ bufferCharData(value, size);
+}
+
+/*
+ * Starts communication with server for COPY FROM BINARY statement
+ */
+static void
+sendBinaryCopyHeader(PGconn *con)
+{
+ char header[] = {'P','G','C','O','P','Y','\n','\377','\r','\n','\0',
+ '\0','\0','\0','\0',
+ '\0','\0','\0','\0' };
+
+ PQputCopyData(con, header, 19);
+}
+
+/*
+ * Finishes communication with server for COPY FROM BINARY statement
+ */
+static void
+sendBinaryCopyTrailer(PGconn *con)
+{
+ static char trailer[] = { 0xFF, 0xFF };
+
+ PQputCopyData(con, trailer, 2);
+}
+
+/*
+ * Flashes current buffer over network if needed
+ */
+static void
+flushBuffer(PGconn *con, PGresult *res, int16_t row_len)
+{
+ if (bin_copy_buffer_length + row_len > BIN_COPY_BUF_SIZE)
+ {
+ /* flush current buffer */
+ if (PQresultStatus(res) == PGRES_COPY_IN)
+ PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
+ bin_copy_buffer_length = 0;
+ }
+}
+
+/*
+ * Sends current branch row to buffer
+ */
+static void
+initBranchBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+{
+ /*
+ * Each row has following extra bytes:
+ * - 2 bytes for number of columns
+ * - 4 bytes as length for each column
+ */
+ int16_t max_row_len = 35 + 2 + 4*3; /* max row size is 32 */
+
+ flushBuffer(con, res, max_row_len);
+
+ addColumnCounter(2);
+
+ addInt32Column(curr + 1);
+ addInt32Column(0);
+}
+
+/*
+ * Sends current teller row to buffer
+ */
+static void
+initTellerBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+{
+ /*
+ * Each row has following extra bytes:
+ * - 2 bytes for number of columns
+ * - 4 bytes as length for each column
+ */
+ int16_t max_row_len = 40 + 2 + 4*4; /* max row size is 40 */
+
+ flushBuffer(con, res, max_row_len);
+
+ addColumnCounter(3);
+
+ addInt32Column(curr + 1);
+ addInt32Column(curr / parent + 1);
+ addInt32Column(0);
+}
+
+/*
+ * Sends current account row to buffer
+ */
+static void
+initAccountBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+{
+ /*
+ * Each row has following extra bytes:
+ * - 2 bytes for number of columns
+ * - 4 bytes as length for each column
+ */
+ int16_t max_row_len = 250 + 2 + 4*4; /* max row size is 250 for int64 */
+
+ flushBuffer(con, res, max_row_len);
+
+ addColumnCounter(3);
+
+ if (scale <= SCALE_32BIT_THRESHOLD)
+ addInt32Column(curr + 1);
+ else
+ addInt64Column(curr);
+
+ addInt32Column(curr / parent + 1);
+ addInt32Column(0);
+}
+
+/*
+ * Universal wrapper for sending data in binary format
+ */
+static void
+initPopulateTableBinary(PGconn *con, char *table, char *columns,
+ int64_t base, initRowMethodBin init_row)
+{
+ int n;
+ PGresult *res;
+ char copy_statement[256];
+ const char *copy_statement_fmt = "copy %s (%s) from stdin (format binary)";
+ int64_t total = base * scale;
+
+ bin_copy_buffer_length = 0;
+
+ /* Use COPY with FREEZE on v14 and later for all ordinary tables */
+ if ((PQserverVersion(con) >= 140000) &&
+ get_table_relkind(con, table) == RELKIND_RELATION)
+ copy_statement_fmt = "copy %s (%s) from stdin with (format binary, freeze on)";
+
+ n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table, columns);
+ if (n >= sizeof(copy_statement))
+ pg_fatal("invalid buffer size: must be at least %d characters long", n);
+ else if (n == -1)
+ pg_fatal("invalid format string");
+
+ res = PQexec(con, copy_statement);
+
+ if (PQresultStatus(res) != PGRES_COPY_IN)
+ pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
+ PQclear(res);
+
+
+ sendBinaryCopyHeader(con);
+
+ for (int64_t i = 0; i < total; i++)
+ {
+ init_row(con, res, i, base);
+ }
+
+ if (PQresultStatus(res) == PGRES_COPY_IN)
+ PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
+ else
+ fprintf(stderr, "Unexpected mode %d instead of %d\n", PQresultStatus(res), PGRES_COPY_IN);
+
+ sendBinaryCopyTrailer(con);
+
+ if (PQresultStatus(res) == PGRES_COPY_IN)
+ {
+ if (PQputCopyEnd(con, NULL) == 1) /* success */
+ {
+ res = PQgetResult(con);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
+ PQclear(res);
+ }
+ else
+ fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
+ }
+}
+
+/*
+ * Wrapper for binary data load
+ */
+static void
+initGenerateDataClientSideBinary(PGconn *con)
+{
+
+ fprintf(stderr, "BINARY mode...\n");
+
+ bin_copy_buffer = pg_malloc(BIN_COPY_BUF_SIZE);
+ bin_copy_buffer_length = 0;
+
+ /*
+ * we do all of this in one transaction to enable the backend's
+ * data-loading optimizations
+ */
+ executeStatement(con, "begin");
+
+ /* truncate away any old data */
+ initTruncateTables(con);
+
+ initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
+ nbranches, initBranchBinary);
+ initPopulateTableBinary(con, "pgbench_tellers", "tid, bid, tbalance",
+ ntellers, initTellerBinary);
+ initPopulateTableBinary(con, "pgbench_accounts", "aid, bid, abalance",
+ naccounts, initAccountBinary);
+
+ executeStatement(con, "commit");
+
+ pg_free(bin_copy_buffer);
+}
+
+/*
+ * Fill the standard tables with some data generated and sent from the client.
+ */
+static void
+initGenerateDataClientSide(PGconn *con)
+{
+ fprintf(stderr, "generating data (client-side) in ");
+
+ switch (data_generation_type)
+ {
+ case GEN_TYPE_COPY_ORIGINAL:
+ initGenerateDataClientSideText(con);
+ break;
+ case GEN_TYPE_COPY_BINARY:
+ initGenerateDataClientSideBinary(con);
+ break;
+ }
+}
+
/*
* Generating data via INSERT .. SELECT .. FROM generate_series
* whole dataset in single transaction
@@ -5500,14 +5872,10 @@ checkInitSteps(const char *initialize_steps)
switch (*step)
{
+ case 'g':
+ case 'C':
case 'G':
- data_init_type++;
- data_generation_type = *step;
- break;
case 'i':
- data_init_type++;
- data_generation_type = *step;
- break;
case 'I':
data_init_type++;
data_generation_type = *step;
@@ -5555,6 +5923,7 @@ runInitSteps(const char *initialize_steps)
initCreateTables(con);
break;
case 'g':
+ case 'C':
op = "client-side generate";
initGenerateDataClientSide(con);
break;
--
2.43.0
From 4aa0ac05765edf6b5f0c13e18ac677287ce78206 Mon Sep 17 00:00:00 2001
From: Fujii Masao <[email protected]>
Date: Fri, 14 Nov 2025 22:40:39 +0900
Subject: [PATCH v11 07/28] pgbench: Fix assertion failure with multiple
\syncpipeline in pipeline mode.
Previously, when pgbench ran a custom script that triggered retriable errors
(e.g., deadlocks) followed by multiple \syncpipeline commands in pipeline mode,
the following assertion failure could occur:
Assertion failed: (res == ((void*)0)), function discardUntilSync, file pgbench.c, line 3594.
The issue was that discardUntilSync() assumed a pipeline sync result
(PGRES_PIPELINE_SYNC) would always be followed by either another sync result
or NULL. This assumption was incorrect: when multiple sync requests were sent,
a sync result could instead be followed by another result type. In such cases,
discardUntilSync() mishandled the results, leading to the assertion failure.
This commit fixes the issue by making discardUntilSync() correctly handle cases
where a pipeline sync result is followed by other result types. It now continues
discarding results until another pipeline sync followed by NULL is reached.
Backpatched to v17, where support for \syncpipeline command in pgbench was
introduced.
Author: Yugo Nagata <[email protected]>
Reviewed-by: Chao Li <[email protected]>
Reviewed-by: Fujii Masao <[email protected]>
Discussion: https://postgr.es/m/[email protected]
Backpatch-through: 17
---
src/bin/pgbench/pgbench.c | 39 ++++++++++++++++++++++++++++-----------
1 file changed, 28 insertions(+), 11 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index d8764ba6fe0..a425176ecdc 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -3563,14 +3563,18 @@ doRetry(CState *st, pg_time_usec_t *now)
}
/*
- * Read results and discard it until a sync point.
+ * Read and discard results until the last sync point.
*/
static int
discardUntilSync(CState *st)
{
bool received_sync = false;
- /* send a sync */
+ /*
+ * Send a Sync message to ensure at least one PGRES_PIPELINE_SYNC is
+ * received and to avoid an infinite loop, since all earlier ones may have
+ * already been received.
+ */
if (!PQpipelineSync(st->con))
{
pg_log_error("client %d aborted: failed to send a pipeline sync",
@@ -3578,29 +3582,42 @@ discardUntilSync(CState *st)
return 0;
}
- /* receive PGRES_PIPELINE_SYNC and null following it */
+ /*
+ * Continue reading results until the last sync point, i.e., until
+ * reaching null just after PGRES_PIPELINE_SYNC.
+ */
for (;;)
{
PGresult *res = PQgetResult(st->con);
+ if (PQstatus(st->con) == CONNECTION_BAD)
+ {
+ pg_log_error("client %d aborted while rolling back the transaction after an error; perhaps the backend died while processing",
+ st->id);
+ PQclear(res);
+ return 0;
+ }
+
if (PQresultStatus(res) == PGRES_PIPELINE_SYNC)
received_sync = true;
- else if (received_sync)
+ else if (received_sync && res == NULL)
{
- /*
- * PGRES_PIPELINE_SYNC must be followed by another
- * PGRES_PIPELINE_SYNC or NULL; otherwise, assert failure.
- */
- Assert(res == NULL);
-
/*
* Reset ongoing sync count to 0 since all PGRES_PIPELINE_SYNC
* results have been discarded.
*/
st->num_syncs = 0;
- PQclear(res);
break;
}
+ else
+ {
+ /*
+ * If a PGRES_PIPELINE_SYNC is followed by something other than
+ * PGRES_PIPELINE_SYNC or NULL, another PGRES_PIPELINE_SYNC will
+ * appear later. Reset received_sync to false to wait for it.
+ */
+ received_sync = false;
+ }
PQclear(res);
}
--
2.43.0
From 9c4f19055597e9adb25e65c2aa8bedf20a09e13d Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Fri, 21 Nov 2025 19:05:58 +0700
Subject: [PATCH v11 08/28] Setting empty string as default value in filler
column
---
src/bin/pgbench/pgbench.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 967f6ce6984..03b5e5c28f0 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -4985,26 +4985,26 @@ initCreateTables(PGconn *con)
static const struct ddlinfo DDLs[] = {
{
"pgbench_history",
- "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22)",
- "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22)",
+ "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22) default ''",
+ "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22) default ''",
0
},
{
"pgbench_tellers",
- "tid int not null,bid int,tbalance int,filler char(84)",
- "tid int not null,bid int,tbalance int,filler char(84)",
+ "tid int not null,bid int,tbalance int,filler char(84) default ''",
+ "tid int not null,bid int,tbalance int,filler char(84) default ''",
1
},
{
"pgbench_accounts",
- "aid int not null,bid int,abalance int,filler char(84)",
- "aid bigint not null,bid int,abalance int,filler char(84)",
+ "aid int not null,bid int,abalance int,filler char(84) default ''",
+ "aid bigint not null,bid int,abalance int,filler char(84) default ''",
1
},
{
"pgbench_branches",
- "bid int not null,bbalance int,filler char(88)",
- "bid int not null,bbalance int,filler char(88)",
+ "bid int not null,bbalance int,filler char(88) default ''",
+ "bid int not null,bbalance int,filler char(88) default ''",
1
}
};
--
2.43.0
From 2aabaa52dffdb78fbefaef95163881c15e18ef29 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <[email protected]>
Date: Fri, 21 Nov 2025 15:03:11 +0200
Subject: [PATCH v11 09/28] Use strtoi64() in pgbench, replacing its open-coded
implementation
Makes the code a little simpler.
The old implementation accepted trailing whitespace, but that was
unnecessary. Firstly, its sibling function for parsing decimals,
strtodouble(), does not accept trailing whitespace. Secondly, none of
the callers can pass a string with trailing whitespace to it.
In the passing, check specifically for ERANGE before printing the "out
of range" error. On some systems, strtoul() and strtod() return EINVAL
on an empty or all-spaces string, and "invalid input syntax" is more
appropriate for that than "out of range". For the existing
strtodouble() function this is purely academical because it's never
called with errorOK==false, but let's be tidy. (Perhaps we should
remove the dead codepaths altogether, but I'll leave that for another
day.)
Reviewed-by: Chao Li <[email protected]>
Reviewed-by: Yuefei Shi <[email protected]>
Reviewed-by: Neil Chen <[email protected]>
Discussion: https://www.postgresql.org/message-id/[email protected]
---
src/bin/pgbench/pgbench.c | 83 +++++++++------------------------------
1 file changed, 19 insertions(+), 64 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index a425176ecdc..68774a59efd 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -982,13 +982,17 @@ usage(void)
progname, progname, PACKAGE_BUGREPORT, PACKAGE_NAME, PACKAGE_URL);
}
-/* return whether str matches "^\s*[-+]?[0-9]+$" */
+/*
+ * Return whether str matches "^\s*[-+]?[0-9]+$"
+ *
+ * This should agree with strtoint64() on what's accepted, ignoring overflows.
+ */
static bool
is_an_int(const char *str)
{
const char *ptr = str;
- /* skip leading spaces; cast is consistent with strtoint64 */
+ /* skip leading spaces */
while (*ptr && isspace((unsigned char) *ptr))
ptr++;
@@ -1012,9 +1016,6 @@ is_an_int(const char *str)
/*
* strtoint64 -- convert a string to 64-bit integer
*
- * This function is a slightly modified version of pg_strtoint64() from
- * src/backend/utils/adt/numutils.c.
- *
* The function returns whether the conversion worked, and if so
* "*result" is set to the result.
*
@@ -1023,71 +1024,25 @@ is_an_int(const char *str)
bool
strtoint64(const char *str, bool errorOK, int64 *result)
{
- const char *ptr = str;
- int64 tmp = 0;
- bool neg = false;
-
- /*
- * Do our own scan, rather than relying on sscanf which might be broken
- * for long long.
- *
- * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
- * value as a negative number.
- */
-
- /* skip leading spaces */
- while (*ptr && isspace((unsigned char) *ptr))
- ptr++;
-
- /* handle sign */
- if (*ptr == '-')
- {
- ptr++;
- neg = true;
- }
- else if (*ptr == '+')
- ptr++;
+ char *end;
- /* require at least one digit */
- if (unlikely(!isdigit((unsigned char) *ptr)))
- goto invalid_syntax;
+ errno = 0;
+ *result = strtoi64(str, &end, 10);
- /* process digits */
- while (*ptr && isdigit((unsigned char) *ptr))
+ if (unlikely(errno == ERANGE))
{
- int8 digit = (*ptr++ - '0');
-
- if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
- unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
- goto out_of_range;
+ if (!errorOK)
+ pg_log_error("value \"%s\" is out of range for type bigint", str);
+ return false;
}
- /* allow trailing whitespace, but not other trailing chars */
- while (*ptr != '\0' && isspace((unsigned char) *ptr))
- ptr++;
-
- if (unlikely(*ptr != '\0'))
- goto invalid_syntax;
-
- if (!neg)
+ if (unlikely(errno != 0 || end == str || *end != '\0'))
{
- if (unlikely(tmp == PG_INT64_MIN))
- goto out_of_range;
- tmp = -tmp;
+ if (!errorOK)
+ pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
+ return false;
}
-
- *result = tmp;
return true;
-
-out_of_range:
- if (!errorOK)
- pg_log_error("value \"%s\" is out of range for type bigint", str);
- return false;
-
-invalid_syntax:
- if (!errorOK)
- pg_log_error("invalid input syntax for type bigint: \"%s\"", str);
- return false;
}
/* convert string to double, detecting overflows/underflows */
@@ -1099,14 +1054,14 @@ strtodouble(const char *str, bool errorOK, double *dv)
errno = 0;
*dv = strtod(str, &end);
- if (unlikely(errno != 0))
+ if (unlikely(errno == ERANGE))
{
if (!errorOK)
pg_log_error("value \"%s\" is out of range for type double", str);
return false;
}
- if (unlikely(end == str || *end != '\0'))
+ if (unlikely(errno != 0 || end == str || *end != '\0'))
{
if (!errorOK)
pg_log_error("invalid input syntax for type double: \"%s\"", str);
--
2.43.0
From dcb85d26f8132eaaf9d096e814b9bda49db7d478 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Fri, 21 Nov 2025 20:06:24 +0700
Subject: [PATCH v11 10/28] Switching COPY FROM BINARY ti run in multiple
transactions
---
src/bin/pgbench/pgbench.c | 27 ++++++++++++++++-----------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 03b5e5c28f0..6b89007a63b 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5496,20 +5496,20 @@ initAccountBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
*/
static void
initPopulateTableBinary(PGconn *con, char *table, char *columns,
- int64_t base, initRowMethodBin init_row)
+ int counter, int64_t base, initRowMethodBin init_row)
{
int n;
PGresult *res;
char copy_statement[256];
const char *copy_statement_fmt = "copy %s (%s) from stdin (format binary)";
- int64_t total = base * scale;
+ int64_t start = base * counter;
bin_copy_buffer_length = 0;
/* Use COPY with FREEZE on v14 and later for all ordinary tables */
if ((PQserverVersion(con) >= 140000) &&
get_table_relkind(con, table) == RELKIND_RELATION)
- copy_statement_fmt = "copy %s (%s) from stdin with (format binary, freeze on)";
+ copy_statement_fmt = "copy %s (%s) from stdin with (format binary)";
n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table, columns);
if (n >= sizeof(copy_statement))
@@ -5526,7 +5526,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
sendBinaryCopyHeader(con);
- for (int64_t i = 0; i < total; i++)
+ for (int64_t i = start; i < start + base; i++)
{
init_row(con, res, i, base);
}
@@ -5573,15 +5573,20 @@ initGenerateDataClientSideBinary(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
- initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
- nbranches, initBranchBinary);
- initPopulateTableBinary(con, "pgbench_tellers", "tid, bid, tbalance",
- ntellers, initTellerBinary);
- initPopulateTableBinary(con, "pgbench_accounts", "aid, bid, abalance",
- naccounts, initAccountBinary);
-
executeStatement(con, "commit");
+ for (int i = 0; i < scale; i++)
+ {
+ initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
+ i, nbranches, initBranchBinary);
+ initPopulateTableBinary(con, "pgbench_tellers", "tid, bid, tbalance",
+ i, ntellers, initTellerBinary);
+ initPopulateTableBinary(con, "pgbench_accounts", "aid, bid, abalance",
+ i, naccounts, initAccountBinary);
+
+ executeStatement(con, "commit");
+ }
+
pg_free(bin_copy_buffer);
}
--
2.43.0
From b8e28881225234fd00b55235bc60fad2dc60b544 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sat, 22 Nov 2025 17:06:00 +0700
Subject: [PATCH v11 11/28] Adding tests for new modes of data generation
---
src/bin/pgbench/pgbench.c | 21 ++++----
src/bin/pgbench/t/001_pgbench_with_server.pl | 52 +++++++++++++++++---
2 files changed, 56 insertions(+), 17 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 6b89007a63b..dd4e5d5e056 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -164,7 +164,7 @@ typedef struct socket_set
#define ALL_INIT_STEPS "dtgCGiIvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
-#define DEFAULT_NXACTS 10 /* default nxacts */
+#define DEFAULT_NXACTS 10 /* default nxacts */
#define MIN_GAUSSIAN_PARAM 2.0 /* minimum parameter for gauss */
@@ -192,7 +192,7 @@ static int scale = 1;
/*
* mode of data generation to use
*/
-static char data_generation_type = '?';
+static char data_generation_type = GEN_TYPE_COPY_ORIGINAL;
/*
* COPY FROM BINARY execution buffer
@@ -4985,26 +4985,26 @@ initCreateTables(PGconn *con)
static const struct ddlinfo DDLs[] = {
{
"pgbench_history",
- "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22) default ''",
- "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22) default ''",
+ "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22) default '?'",
+ "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22) default '?'",
0
},
{
"pgbench_tellers",
- "tid int not null,bid int,tbalance int,filler char(84) default ''",
- "tid int not null,bid int,tbalance int,filler char(84) default ''",
+ "tid int not null,bid int,tbalance int,filler char(84)",
+ "tid int not null,bid int,tbalance int,filler char(84)",
1
},
{
"pgbench_accounts",
- "aid int not null,bid int,abalance int,filler char(84) default ''",
- "aid bigint not null,bid int,abalance int,filler char(84) default ''",
+ "aid int not null,bid int,abalance int,filler char(84) default '?'",
+ "aid bigint not null,bid int,abalance int,filler char(84) default '?'",
1
},
{
"pgbench_branches",
- "bid int not null,bbalance int,filler char(88) default ''",
- "bid int not null,bbalance int,filler char(88) default ''",
+ "bid int not null,bbalance int,filler char(88)",
+ "bid int not null,bbalance int,filler char(88)",
1
}
};
@@ -7837,6 +7837,7 @@ main(int argc, char **argv)
}
}
+ checkInitSteps(initialize_steps);
runInitSteps(initialize_steps);
exit(0);
}
diff --git a/src/bin/pgbench/t/001_pgbench_with_server.pl b/src/bin/pgbench/t/001_pgbench_with_server.pl
index 581e9af7907..a377048ead1 100644
--- a/src/bin/pgbench/t/001_pgbench_with_server.pl
+++ b/src/bin/pgbench/t/001_pgbench_with_server.pl
@@ -16,25 +16,30 @@ sub check_data_state
local $Test::Builder::Level = $Test::Builder::Level + 1;
my $node = shift;
my $type = shift;
+ my $sql_result;
- my $sql_result = $node->safe_psql('postgres',
- 'SELECT count(*) AS null_count FROM pgbench_accounts WHERE filler IS NULL LIMIT 10;'
- );
- is($sql_result, '0',
- "$type: filler column of pgbench_accounts has no NULL data");
$sql_result = $node->safe_psql('postgres',
'SELECT count(*) AS null_count FROM pgbench_branches WHERE filler IS NULL;'
);
is($sql_result, '1',
"$type: filler column of pgbench_branches has only NULL data");
+
$sql_result = $node->safe_psql('postgres',
'SELECT count(*) AS null_count FROM pgbench_tellers WHERE filler IS NULL;'
);
is($sql_result, '10',
"$type: filler column of pgbench_tellers has only NULL data");
+
+ $sql_result = $node->safe_psql('postgres',
+ 'SELECT count(*) AS null_count FROM pgbench_accounts WHERE filler IS NULL LIMIT 10;'
+ );
+ is($sql_result, '0',
+ "$type: filler column of pgbench_accounts has no NULL data");
+
$sql_result = $node->safe_psql('postgres',
'SELECT count(*) AS data_count FROM pgbench_history;');
- is($sql_result, '0', "$type: pgbench_history has no data");
+ is($sql_result, '0',
+ "$type: pgbench_history has no data");
}
# start a pgbench specific server
@@ -125,7 +130,7 @@ $node->pgbench(
'pgbench scale 1 initialization',);
# Check data state, after client-side data generation.
-check_data_state($node, 'client-side');
+check_data_state($node, 'client-side (default options)');
# Again, with all possible options
$node->pgbench(
@@ -143,6 +148,7 @@ $node->pgbench(
qr{done in \d+\.\d\d s }
],
'pgbench scale 1 initialization');
+check_data_state($node, 'client-side (all options)');
# Test interaction of --init-steps with legacy step-selection options
$node->pgbench(
@@ -164,6 +170,38 @@ $node->pgbench(
# Check data state, after server-side data generation.
check_data_state($node, 'server-side');
+# Test server-side generation with UNNEST
+$node->pgbench(
+ '--initialize --init-steps=dtI',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side UNNEST');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (unnest)');
+
+# Test server-side generation with UNNEST
+$node->pgbench(
+ '--initialize --init-steps=dtC',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side BINARY');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (binary)');
+
# Run all builtin scripts, for a few transactions each
$node->pgbench(
'--transactions=5 -Dfoo=bla --client=2 --protocol=simple --builtin=t'
--
2.43.0
From b3f2bce34232d299abc7644a8579f0ce49c8c9d6 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 23 Nov 2025 14:05:59 +0700
Subject: [PATCH v11 12/28] Fixing compiler warnings about unused procedures by
removing or commenting them out as they might be needed a bit later
---
src/bin/pgbench/pgbench.c | 31 +++++--------------------------
1 file changed, 5 insertions(+), 26 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 0a3ba21dcc9..682db61ff61 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5201,7 +5201,7 @@ initGenerateDataClientSideText(PGconn *con)
/*
* Dumps binary buffer to file (purely for debugging)
- */
+ *
static void
dumpBufferToFile(char *filename)
{
@@ -5226,6 +5226,7 @@ dumpBufferToFile(char *filename)
fclose(file_ptr);
}
+ */
/*
* Save char data to buffer
@@ -5271,37 +5272,14 @@ addColumnCounter(int16_t n)
/*
* adds column with NULL value
- */
+ *
static void
addNullColumn()
{
int32_t null = -1;
bufferData((void *) &null, sizeof(null));
}
-
-/*
- * adds column with int8 value
*/
-static void
-addInt8Column(int8_t value)
-{
- int8_t data = value;
- int32_t size = sizeof(data);
- bufferData((void *) &size, sizeof(size));
- bufferData((void *) &data, sizeof(data));
-}
-
-/*
- * adds column with int16 value
- */
-static void
-addInt16Column(int16_t value)
-{
- int16_t data = value;
- int32_t size = sizeof(data);
- bufferData((void *) &size, sizeof(size));
- bufferData((void *) &data, sizeof(data));
-}
/*
* adds column with inti32 value
@@ -5329,7 +5307,7 @@ addInt64Column(int64_t value)
/*
* adds column with char value
- */
+ *
static void
addCharColumn(char *value)
{
@@ -5337,6 +5315,7 @@ addCharColumn(char *value)
bufferData((void *) &size, sizeof(size));
bufferCharData(value, size);
}
+ */
/*
* Starts communication with server for COPY FROM BINARY statement
--
2.43.0
From 9ab7fe302ba7de40593e7cba8ea1ca3b876c1ea5 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Thu, 29 Jan 2026 19:22:16 +0700
Subject: [PATCH v11 13/28] Moving PQclear call to the end of procedure to
avoid access to previously released memory
---
src/bin/pgbench/pgbench.c | 27 +++++++++++++++++----------
1 file changed, 17 insertions(+), 10 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 682db61ff61..3687b65871e 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5363,11 +5363,11 @@ static void
initBranchBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
{
/*
- * Each row has following extra bytes:
+ * Each row of branch table has following extra bytes:
* - 2 bytes for number of columns
- * - 4 bytes as length for each column
+ * - 4 bytes as length for each of table's 3 columns
*/
- int16_t max_row_len = 35 + 2 + 4*3; /* max row size is 32 */
+ int16_t max_row_len = 35 + 2 + 4*3; /* max row size is 35 */
flushBuffer(con, res, max_row_len);
@@ -5384,9 +5384,9 @@ static void
initTellerBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
{
/*
- * Each row has following extra bytes:
+ * Each row of tellers table has following extra bytes:
* - 2 bytes for number of columns
- * - 4 bytes as length for each column
+ * - 4 bytes as length for each of table's 4 columns
*/
int16_t max_row_len = 40 + 2 + 4*4; /* max row size is 40 */
@@ -5406,9 +5406,9 @@ static void
initAccountBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
{
/*
- * Each row has following extra bytes:
+ * Each row of accounts table has following extra bytes:
* - 2 bytes for number of columns
- * - 4 bytes as length for each column
+ * - 4 bytes as length for each of table's 4 columns
*/
int16_t max_row_len = 250 + 2 + 4*4; /* max row size is 250 for int64 */
@@ -5455,11 +5455,11 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
if (PQresultStatus(res) != PGRES_COPY_IN)
pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
- PQclear(res);
sendBinaryCopyHeader(con);
+
for (int64_t i = start; i < start + base; i++)
{
init_row(con, res, i, base);
@@ -5470,8 +5470,10 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
else
fprintf(stderr, "Unexpected mode %d instead of %d\n", PQresultStatus(res), PGRES_COPY_IN);
+
sendBinaryCopyTrailer(con);
+
if (PQresultStatus(res) == PGRES_COPY_IN)
{
if (PQputCopyEnd(con, NULL) == 1) /* success */
@@ -5484,6 +5486,8 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
else
fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
}
+
+ PQclear(res);
}
/*
@@ -5499,8 +5503,9 @@ initGenerateDataClientSideBinary(PGconn *con)
bin_copy_buffer_length = 0;
/*
- * we do all of this in one transaction to enable the backend's
- * data-loading optimizations
+ * we do all of this in multiple transactions
+ * to minimize load on DB server and perhaps
+ * in future allow load in parallel sessions
*/
executeStatement(con, "begin");
@@ -5511,6 +5516,8 @@ initGenerateDataClientSideBinary(PGconn *con)
for (int i = 0; i < scale; i++)
{
+ executeStatement(con, "begin");
+
initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
i, nbranches, initBranchBinary);
initPopulateTableBinary(con, "pgbench_tellers", "tid, bid, tbalance",
--
2.43.0
From 3d7b2c53b3fa45b8848369bfbc49a0c7e5854304 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Thu, 29 Jan 2026 19:47:26 +0700
Subject: [PATCH v11 14/28] Fixing error about freeing memory twice
---
src/bin/pgbench/pgbench.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 3687b65871e..6f42438d970 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5462,6 +5462,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
for (int64_t i = start; i < start + base; i++)
{
+ res = PQgetResult(con);
init_row(con, res, i, base);
}
@@ -5473,7 +5474,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
sendBinaryCopyTrailer(con);
-
+ res = PQgetResult(con);
if (PQresultStatus(res) == PGRES_COPY_IN)
{
if (PQputCopyEnd(con, NULL) == 1) /* success */
@@ -5481,7 +5482,6 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
res = PQgetResult(con);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
- PQclear(res);
}
else
fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
--
2.43.0
From a71c90f556ba143335d9631ebbdf96b7120c6d1d Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Thu, 29 Jan 2026 20:24:54 +0700
Subject: [PATCH v11 15/28] Fixing memory leak during data flush in COPY BINARY
test
---
src/bin/pgbench/pgbench.c | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 6f42438d970..8eca537b718 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -871,7 +871,7 @@ static bool socket_has_input(socket_set *sa, int fd, int idx);
/* callback used to build rows for COPY during data loading */
typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
-typedef void (*initRowMethodBin) (PGconn *con, PGresult *res, int64_t curr, int32_t parent);
+typedef void (*initRowMethodBin) (PGconn *con, int64_t curr, int32_t parent);
/* callback functions for our flex lexer */
static const PsqlScanCallbacks pgbench_callbacks = {
@@ -5345,13 +5345,18 @@ sendBinaryCopyTrailer(PGconn *con)
* Flashes current buffer over network if needed
*/
static void
-flushBuffer(PGconn *con, PGresult *res, int16_t row_len)
+flushBuffer(PGconn *con, int16_t row_len)
{
+ PGresult *res;
+
if (bin_copy_buffer_length + row_len > BIN_COPY_BUF_SIZE)
{
+ res = PQgetResult(con);
/* flush current buffer */
if (PQresultStatus(res) == PGRES_COPY_IN)
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
+ else
+ pg_fatal("It is NOT a COPY command that is currently running");
bin_copy_buffer_length = 0;
}
}
@@ -5360,7 +5365,7 @@ flushBuffer(PGconn *con, PGresult *res, int16_t row_len)
* Sends current branch row to buffer
*/
static void
-initBranchBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+initBranchBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*
* Each row of branch table has following extra bytes:
@@ -5369,7 +5374,7 @@ initBranchBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
*/
int16_t max_row_len = 35 + 2 + 4*3; /* max row size is 35 */
- flushBuffer(con, res, max_row_len);
+ flushBuffer(con, max_row_len);
addColumnCounter(2);
@@ -5381,7 +5386,7 @@ initBranchBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
* Sends current teller row to buffer
*/
static void
-initTellerBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+initTellerBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*
* Each row of tellers table has following extra bytes:
@@ -5390,7 +5395,7 @@ initTellerBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
*/
int16_t max_row_len = 40 + 2 + 4*4; /* max row size is 40 */
- flushBuffer(con, res, max_row_len);
+ flushBuffer(con, max_row_len);
addColumnCounter(3);
@@ -5403,7 +5408,7 @@ initTellerBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
* Sends current account row to buffer
*/
static void
-initAccountBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
+initAccountBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*
* Each row of accounts table has following extra bytes:
@@ -5412,7 +5417,7 @@ initAccountBinary(PGconn *con, PGresult *res, int64_t curr, int32_t parent)
*/
int16_t max_row_len = 250 + 2 + 4*4; /* max row size is 250 for int64 */
- flushBuffer(con, res, max_row_len);
+ flushBuffer(con, max_row_len);
addColumnCounter(3);
@@ -5462,10 +5467,10 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
for (int64_t i = start; i < start + base; i++)
{
- res = PQgetResult(con);
- init_row(con, res, i, base);
+ init_row(con, i, base);
}
+ res = PQgetResult(con);
if (PQresultStatus(res) == PGRES_COPY_IN)
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
else
@@ -5474,6 +5479,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
sendBinaryCopyTrailer(con);
+
res = PQgetResult(con);
if (PQresultStatus(res) == PGRES_COPY_IN)
{
--
2.43.0
From a5933f40be0f3d408bb920f296fb5119c3ec28f4 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Fri, 30 Jan 2026 13:59:36 +0700
Subject: [PATCH v11 16/28] Fixing memory leak shown by valgrind
---
src/bin/pgbench/pgbench.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 8eca537b718..466d9023d72 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5357,6 +5357,8 @@ flushBuffer(PGconn *con, int16_t row_len)
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
else
pg_fatal("It is NOT a COPY command that is currently running");
+
+ PQclear(res);
bin_copy_buffer_length = 0;
}
}
@@ -5460,6 +5462,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
if (PQresultStatus(res) != PGRES_COPY_IN)
pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
+ PQclear(res);
sendBinaryCopyHeader(con);
@@ -5475,6 +5478,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
else
fprintf(stderr, "Unexpected mode %d instead of %d\n", PQresultStatus(res), PGRES_COPY_IN);
+ PQclear(res);
sendBinaryCopyTrailer(con);
@@ -5485,6 +5489,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
{
if (PQputCopyEnd(con, NULL) == 1) /* success */
{
+ PQclear(res);
res = PQgetResult(con);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
@@ -5492,7 +5497,6 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
else
fprintf(stderr, "Error: %s\n", PQerrorMessage(con));
}
-
PQclear(res);
}
--
2.43.0
From 6a3036f898b050d5e71e70eaceccd63003aa6444 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Fri, 6 Feb 2026 13:53:07 +0700
Subject: [PATCH v11 17/28] Adding ability to switch data init between single
and multiple transactions
---
src/bin/pgbench/pgbench.c | 192 +++++++++----------
src/bin/pgbench/t/001_pgbench_with_server.pl | 93 ++++++++-
2 files changed, 173 insertions(+), 112 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 466d9023d72..af71e358e71 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -159,9 +159,8 @@ typedef struct socket_set
/********************************************************************
* some configurable parameters */
-
-#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
-#define ALL_INIT_STEPS "dtgCGiIvpf" /* all possible steps */
+#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
+#define ALL_INIT_STEPS "dtgMScGUvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
#define DEFAULT_NXACTS 10 /* default nxacts */
@@ -171,14 +170,17 @@ typedef struct socket_set
#define MIN_ZIPFIAN_PARAM 1.001 /* minimum parameter for zipfian */
#define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
-/* original single transaction server-side method */
-#define GEN_TYPE_INSERT_ORIGINAL 'G' /* use INSERT .. SELECT generate_series to generate data */
-/* 'one transaction per scale' server-side methods */
-#define GEN_TYPE_INSERT_SERIES 'i' /* use INSERT .. SELECT generate_series to generate data */
-#define GEN_TYPE_INSERT_UNNEST 'I' /* use INSERT .. SELECT unnest to generate data */
-#define GEN_TYPE_COPY_ORIGINAL 'g' /* use COPY .. FROM STDIN .. TEXT to generate data */
-#define GEN_TYPE_COPY_BINARY 'C' /* use COPY .. FROM STDIN .. BINARY to generate data */
-
+/* server-side methods to generate data */
+#define INIT_STEP_GEN_TYPE_INSERT_SERIES 'G' /* use INSERT .. SELECT generate_series to generate data */
+#define INIT_STEP_GEN_TYPE_INSERT_UNNEST 'U' /* use INSERT .. SELECT unnest to generate data */
+/* client-side methods to generate data */
+#define INIT_STEP_GEN_TYPE_COPY_TEXT 'g' /* use COPY .. FROM STDIN .. TEXT to generate data */
+#define INIT_STEP_GEN_TYPE_COPY_BINARY 'c' /* use COPY .. FROM STDIN .. BINARY to generate data */
+/* data init pseudo steps */
+#define INIT_STEP_GEN_TYPE_SINGLE_XACT 'S' /* switch to init data as single transaction */
+#define INIT_STEP_GEN_TYPE_MULTI_XACT 'M' /* switch to init data as multiple transactions */
+
+static bool multi_xact = false; /* init data type (as single or multiple transactions) */
static int nxacts = 0; /* number of transactions per client */
static int duration = 0; /* duration in seconds */
static int64 end_time = 0; /* when to stop in micro seconds, under -T */
@@ -192,7 +194,7 @@ static int scale = 1;
/*
* mode of data generation to use
*/
-static char data_generation_type = GEN_TYPE_COPY_ORIGINAL;
+static char data_generation_type = INIT_STEP_GEN_TYPE_COPY_TEXT;
/*
* COPY FROM BINARY execution buffer
@@ -934,11 +936,13 @@ usage(void)
" run selected initialization steps, in the specified order\n"
" d: drop any existing pgbench tables\n"
" t: create the tables used by the standard pgbench scenario\n"
- " g: generate data, client-side\n"
- " C: client-side (single TNX) COPY .. FROM STDIN .. BINARY\n"
- " G: generate data, server-side in single transaction\n"
- " i: server-side (multiple TXNs) INSERT .. SELECT generate_series\n"
- " I: server-side (multiple TXNs) INSERT .. SELECT unnest\n"
+ " to generate data, client-side:\n"
+ " g: COPY .. FROM STDIN .. TEXT\n"
+ " c: COPY .. FROM STDIN .. BINARY\n"
+ " to generate data, server-side:\n"
+ " G: INSERT .. SELECT generate_series\n"
+ " U: INSERT .. SELECT unnest\n"
+ " M: use multiple transactions to initialize data\n"
" v: invoke VACUUM on the standard tables\n"
" p: create primary key indexes on the standard tables\n"
" f: create foreign keys between the standard tables\n"
@@ -5049,8 +5053,8 @@ initAccount(PQExpBufferData *sql, int64 curr)
}
static void
-initPopulateTable(PGconn *con, const char *table, int64 base,
- initRowMethod init_row)
+initPopulateTableText(PGconn *con, const char *table, int64 base,
+ initRowMethod init_row)
{
int n;
int64 k;
@@ -5178,6 +5182,9 @@ initGenerateDataClientSideText(PGconn *con)
{
fprintf(stderr, "TEXT mode...\n");
+ if (multi_xact)
+ fprintf(stderr, "WARNING! Multiple transactions are not supported in this mode\n");
+
/*
* we do all of this in one transaction to enable the backend's
* data-loading optimizations
@@ -5191,9 +5198,9 @@ initGenerateDataClientSideText(PGconn *con)
* fill branches, tellers, accounts in that order in case foreign keys
* already exist
*/
- initPopulateTable(con, "pgbench_branches", nbranches, initBranch);
- initPopulateTable(con, "pgbench_tellers", ntellers, initTeller);
- initPopulateTable(con, "pgbench_accounts", naccounts, initAccount);
+ initPopulateTableText(con, "pgbench_branches", nbranches, initBranch);
+ initPopulateTableText(con, "pgbench_tellers", ntellers, initTeller);
+ initPopulateTableText(con, "pgbench_accounts", naccounts, initAccount);
executeStatement(con, "commit");
}
@@ -5450,7 +5457,8 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
/* Use COPY with FREEZE on v14 and later for all ordinary tables */
if ((PQserverVersion(con) >= 140000) &&
get_table_relkind(con, table) == RELKIND_RELATION)
- copy_statement_fmt = "copy %s (%s) from stdin with (format binary)";
+ if (!multi_xact)
+ copy_statement_fmt = "copy %s (%s) from stdin with (format binary, freeze on)";
n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table, columns);
if (n >= sizeof(copy_statement))
@@ -5522,11 +5530,13 @@ initGenerateDataClientSideBinary(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
for (int i = 0; i < scale; i++)
{
- executeStatement(con, "begin");
+ if (multi_xact)
+ executeStatement(con, "begin");
initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
i, nbranches, initBranchBinary);
@@ -5535,9 +5545,13 @@ initGenerateDataClientSideBinary(PGconn *con)
initPopulateTableBinary(con, "pgbench_accounts", "aid, bid, abalance",
i, naccounts, initAccountBinary);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
}
+ if (!multi_xact)
+ executeStatement(con, "commit");
+
pg_free(bin_copy_buffer);
}
@@ -5547,14 +5561,15 @@ initGenerateDataClientSideBinary(PGconn *con)
static void
initGenerateDataClientSide(PGconn *con)
{
- fprintf(stderr, "generating data (client-side) in ");
+ fprintf(stderr, "generating data (client-side as %s transaction%s) in ",
+ multi_xact ? "multiple" : "single", multi_xact ? "s" : "");
switch (data_generation_type)
{
- case GEN_TYPE_COPY_ORIGINAL:
+ case INIT_STEP_GEN_TYPE_COPY_TEXT:
initGenerateDataClientSideText(con);
break;
- case GEN_TYPE_COPY_BINARY:
+ case INIT_STEP_GEN_TYPE_COPY_BINARY:
initGenerateDataClientSideBinary(con);
break;
}
@@ -5562,58 +5577,7 @@ initGenerateDataClientSide(PGconn *con)
/*
* Generating data via INSERT .. SELECT .. FROM generate_series
- * whole dataset in single transaction
- */
-static void
-generateDataInsertSingleTXN(PGconn *con)
-{
- PQExpBufferData sql;
-
- fprintf(stderr, "via INSERT .. SELECT generate_series... in single TXN\n");
-
-
- /*
- * we do all of this in one transaction to enable the backend's
- * data-loading optimizations
- */
- executeStatement(con, "begin");
-
- /* truncate away any old data */
- initTruncateTables(con);
-
- initPQExpBuffer(&sql);
-
- printfPQExpBuffer(&sql,
- "insert into pgbench_branches(bid, bbalance) "
- "select bid, 0 "
- "from generate_series(1, %d) as bid",
- scale * nbranches);
- executeStatement(con, sql.data);
-
- printfPQExpBuffer(&sql,
- "insert into pgbench_tellers(tid, bid, tbalance) "
- "select tid + 1, tid / %d + 1, 0 "
- "from generate_series(0, %d) as tid",
- ntellers, (scale * ntellers) - 1);
- executeStatement(con, sql.data);
-
- printfPQExpBuffer(&sql,
- "insert into pgbench_accounts(aid, bid, abalance, "
- "filler) "
- "select aid + 1, aid / %d + 1, 0, '' "
- "from generate_series(0, " INT64_FORMAT ") as aid",
- naccounts, (int64) (scale * naccounts) - 1);
- executeStatement(con, sql.data);
-
- executeStatement(con, "commit");
-
- termPQExpBuffer(&sql);
-}
-
-
-/*
- * Generating data via INSERT .. SELECT .. FROM generate_series
- * One transaction per 'scale'
+ * Possibly as "One transaction per scale" in multi-transaction mode
*/
static void
generateDataInsertSeries(PGconn *con)
@@ -5629,11 +5593,13 @@ generateDataInsertSeries(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
for (int i = 0; i < scale; i++)
{
- executeStatement(con, "begin");
+ if (multi_xact)
+ executeStatement(con, "begin");
printfPQExpBuffer(&sql,
"insert into pgbench_branches(bid, bbalance) "
@@ -5657,15 +5623,19 @@ generateDataInsertSeries(PGconn *con)
(int64) (i + 1) * naccounts - 1);
executeStatement(con, sql.data);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
}
+ if (!multi_xact)
+ executeStatement(con, "commit");
+
termPQExpBuffer(&sql);
}
/*
* Generating data via INSERT .. SELECT .. FROM unnest
- * One transaction per 'scale'
+ * Possibly as "One transaction per scale" in multi-tansaction mode
*/
static void
generateDataInsertUnnest(PGconn *con)
@@ -5681,11 +5651,13 @@ generateDataInsertUnnest(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
for (int s = 0; s < scale; s++)
{
- executeStatement(con, "begin");
+ if (multi_xact)
+ executeStatement(con, "begin");
printfPQExpBuffer(&sql,
"insert into pgbench_branches(bid,bbalance) "
@@ -5714,14 +5686,18 @@ generateDataInsertUnnest(PGconn *con)
(int64) (s + 1) * naccounts, naccounts);
executeStatement(con, sql.data);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
}
+ if (!multi_xact)
+ executeStatement(con, "commit");
+
termPQExpBuffer(&sql);
}
/*
- * Fill the standard tables with some data generated on the server
+ * Fill the standard tables with some data generated on the server side
*
* As already the case with the client-side data generation, the filler
* column defaults to NULL in pgbench_branches and pgbench_tellers,
@@ -5730,17 +5706,15 @@ generateDataInsertUnnest(PGconn *con)
static void
initGenerateDataServerSide(PGconn *con)
{
- fprintf(stderr, "generating data (server-side) ");
+ fprintf(stderr, "generating data (server-side as %s transaction%s) ",
+ multi_xact ? "multiple" : "single", multi_xact ? "s" : "");
switch (data_generation_type)
{
- case GEN_TYPE_INSERT_ORIGINAL:
- generateDataInsertSingleTXN(con);
- break;
- case GEN_TYPE_INSERT_SERIES:
+ case INIT_STEP_GEN_TYPE_INSERT_SERIES:
generateDataInsertSeries(con);
break;
- case GEN_TYPE_INSERT_UNNEST:
+ case INIT_STEP_GEN_TYPE_INSERT_UNNEST:
generateDataInsertUnnest(con);
break;
}
@@ -5845,19 +5819,20 @@ checkInitSteps(const char *initialize_steps)
switch (*step)
{
- case 'g':
- case 'C':
- case 'G':
- case 'i':
- case 'I':
+ case INIT_STEP_GEN_TYPE_COPY_TEXT:
+ case INIT_STEP_GEN_TYPE_COPY_BINARY:
+ case INIT_STEP_GEN_TYPE_INSERT_SERIES:
+ case INIT_STEP_GEN_TYPE_INSERT_UNNEST:
data_init_type++;
data_generation_type = *step;
break;
}
}
+ if (data_init_type == 0)
+ pg_log_error("WARNING! No data generation type is provided");
if (data_init_type > 1)
- pg_log_error("WARNING! More than one type of server-side data generation is requested");
+ pg_log_error("WARNING! More than one type of data initialization is requested");
}
/*
@@ -5895,17 +5870,22 @@ runInitSteps(const char *initialize_steps)
op = "create tables";
initCreateTables(con);
break;
- case 'g':
- case 'C':
+ case INIT_STEP_GEN_TYPE_COPY_TEXT:
+ case INIT_STEP_GEN_TYPE_COPY_BINARY:
op = "client-side generate";
initGenerateDataClientSide(con);
break;
- case 'G':
- case 'i':
- case 'I':
+ case INIT_STEP_GEN_TYPE_INSERT_SERIES:
+ case INIT_STEP_GEN_TYPE_INSERT_UNNEST:
op = "server-side generate";
initGenerateDataServerSide(con);
break;
+ case INIT_STEP_GEN_TYPE_SINGLE_XACT:
+ multi_xact = false;
+ break;
+ case INIT_STEP_GEN_TYPE_MULTI_XACT:
+ multi_xact = true;
+ break;
case 'v':
op = "vacuum";
initVacuum(con);
diff --git a/src/bin/pgbench/t/001_pgbench_with_server.pl b/src/bin/pgbench/t/001_pgbench_with_server.pl
index a377048ead1..acb1e31d3e8 100644
--- a/src/bin/pgbench/t/001_pgbench_with_server.pl
+++ b/src/bin/pgbench/t/001_pgbench_with_server.pl
@@ -117,6 +117,7 @@ $node->pgbench(
[qr{Perhaps you need to do initialization}],
'run without init');
+
# Initialize pgbench tables scale 1
$node->pgbench(
'-i', 0,
@@ -160,7 +161,7 @@ $node->pgbench(
qr{creating tables},
qr{creating 3 partitions},
qr{creating primary keys},
- qr{generating data \(server-side\)},
+ qr{generating data \(server-side as single transaction\)},
qr{creating foreign keys},
qr{(?!vacuuming)}, # no vacuum
qr{done in \d+\.\d\d s }
@@ -170,15 +171,16 @@ $node->pgbench(
# Check data state, after server-side data generation.
check_data_state($node, 'server-side');
+
# Test server-side generation with UNNEST
$node->pgbench(
- '--initialize --init-steps=dtI',
+ '--initialize --init-steps=dtU',
0,
[qr{^$}],
[
qr{dropping old tables},
qr{creating tables},
- qr{generating data \(server-side\)},
+ qr{generating data \(server-side as single transaction\)},
qr{done in \d+\.\d\d s }
],
'pgbench --init-steps server-side UNNEST');
@@ -186,15 +188,48 @@ $node->pgbench(
# Check data state, after server-side data generation.
check_data_state($node, 'server-side (unnest)');
-# Test server-side generation with UNNEST
+
+# Test server-side generation with COPY TEXT
+$node->pgbench(
+ '--initialize --init-steps=dtg',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as single transaction},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side TEXT');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (text)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMg',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as multiple transactions},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side TEXT');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (text)');
+
+
+# Test server-side generation with COPY BINARY
$node->pgbench(
- '--initialize --init-steps=dtC',
+ '--initialize --init-steps=dtc',
0,
[qr{^$}],
[
qr{dropping old tables},
qr{creating tables},
- qr{generating data \(client-side\)},
+ qr{generating data \(client-side as single transaction},
qr{done in \d+\.\d\d s }
],
'pgbench --init-steps client-side BINARY');
@@ -202,6 +237,52 @@ $node->pgbench(
# Check data state, after server-side data generation.
check_data_state($node, 'client-side (binary)');
+$node->pgbench(
+ '--initialize --init-steps=dtSc',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as single transaction},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side BINARY');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (binary)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMc',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as multiple transactions},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side BINARY');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (binary)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMc',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as multiple transactions},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side BINARY');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'client-side (binary)');
+
+
# Run all builtin scripts, for a few transactions each
$node->pgbench(
'--transactions=5 -Dfoo=bla --client=2 --protocol=simple --builtin=t'
--
2.43.0
From 2471bb47bf715402ece61b084b69afe3bb53742e Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Fri, 6 Feb 2026 13:55:47 +0700
Subject: [PATCH v11 18/28] Removing commented out procedures
---
src/bin/pgbench/pgbench.c | 52 ---------------------------------------
1 file changed, 52 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index af71e358e71..34464b7037c 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5206,35 +5206,6 @@ initGenerateDataClientSideText(PGconn *con)
}
-/*
- * Dumps binary buffer to file (purely for debugging)
- *
-static void
-dumpBufferToFile(char *filename)
-{
- FILE *file_ptr;
- size_t bytes_written;
-
- file_ptr = fopen(filename, "wb");
- if (file_ptr == NULL)
- {
- fprintf(stderr, "Error opening file %s\n", filename);
- return; // EXIT_FAILURE;
- }
-
- bytes_written = fwrite(bin_copy_buffer, 1, bin_copy_buffer_length, file_ptr);
-
- if (bytes_written != bin_copy_buffer_length)
- {
- fprintf(stderr, "Error writing to file or incomplete write\n");
- fclose(file_ptr);
- return; // EXIT_FAILURE;
- }
-
- fclose(file_ptr);
-}
- */
-
/*
* Save char data to buffer
*/
@@ -5277,17 +5248,6 @@ addColumnCounter(int16_t n)
bufferData((void *) &n, sizeof(n));
}
-/*
- * adds column with NULL value
- *
-static void
-addNullColumn()
-{
- int32_t null = -1;
- bufferData((void *) &null, sizeof(null));
-}
- */
-
/*
* adds column with inti32 value
*/
@@ -5312,18 +5272,6 @@ addInt64Column(int64_t value)
bufferData((void *) &data, sizeof(data));
}
-/*
- * adds column with char value
- *
-static void
-addCharColumn(char *value)
-{
- int32_t size = strlen(value);
- bufferData((void *) &size, sizeof(size));
- bufferCharData(value, size);
-}
- */
-
/*
* Starts communication with server for COPY FROM BINARY statement
*/
--
2.43.0
From 10dfa622b09c420d79da8774c9321f91540c8600 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 16 Feb 2026 09:08:17 +0700
Subject: [PATCH v11 19/28] Small changes due to big code review
---
src/bin/pgbench/pgbench.c | 71 +++++++++++++++++++++------------------
1 file changed, 39 insertions(+), 32 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 34464b7037c..9f3d7f99a25 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -873,7 +873,7 @@ static bool socket_has_input(socket_set *sa, int fd, int idx);
/* callback used to build rows for COPY during data loading */
typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
-typedef void (*initRowMethodBin) (PGconn *con, int64_t curr, int32_t parent);
+typedef void (*initRowMethodBinary) (PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter);
/* callback functions for our flex lexer */
static const PsqlScanCallbacks pgbench_callbacks = {
@@ -942,7 +942,8 @@ usage(void)
" to generate data, server-side:\n"
" G: INSERT .. SELECT generate_series\n"
" U: INSERT .. SELECT unnest\n"
- " M: use multiple transactions to initialize data\n"
+ " S: flag to use single transaction to initialize data\n"
+ " M: flag to use multiple transactions to initialize data\n"
" v: invoke VACUUM on the standard tables\n"
" p: create primary key indexes on the standard tables\n"
" f: create foreign keys between the standard tables\n"
@@ -5053,8 +5054,8 @@ initAccount(PQExpBufferData *sql, int64 curr)
}
static void
-initPopulateTableText(PGconn *con, const char *table, int64 base,
- initRowMethod init_row)
+initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
+ initRowMethod init_row)
{
int n;
int64 k;
@@ -5178,7 +5179,7 @@ initPopulateTableText(PGconn *con, const char *table, int64 base,
* a blank-padded string in pgbench_accounts.
*/
static void
-initGenerateDataClientSideText(PGconn *con)
+initGenerateDataClientSideTextFrmt(PGconn *con)
{
fprintf(stderr, "TEXT mode...\n");
@@ -5198,9 +5199,9 @@ initGenerateDataClientSideText(PGconn *con)
* fill branches, tellers, accounts in that order in case foreign keys
* already exist
*/
- initPopulateTableText(con, "pgbench_branches", nbranches, initBranch);
- initPopulateTableText(con, "pgbench_tellers", ntellers, initTeller);
- initPopulateTableText(con, "pgbench_accounts", naccounts, initAccount);
+ initPopulateTableCopyText(con, "pgbench_branches", nbranches, initBranch);
+ initPopulateTableCopyText(con, "pgbench_tellers", ntellers, initTeller);
+ initPopulateTableCopyText(con, "pgbench_accounts", naccounts, initAccount);
executeStatement(con, "commit");
}
@@ -5212,6 +5213,8 @@ initGenerateDataClientSideText(PGconn *con)
static void
bufferCharData(char *src, int32_t len)
{
+ Assert(bin_copy_buffer_length + len <= BIN_COPY_BUF_SIZE);
+
memcpy((char *) bin_copy_buffer + bin_copy_buffer_length, (char *) src, len);
bin_copy_buffer_length += len;
}
@@ -5227,13 +5230,16 @@ bufferData(void *src, int32_t len)
bufferCharData(src, len);
#else
if (len == 1)
+ {
bufferCharData(src, len);
- else
- for (int32_t i = 0; i < len; i++)
- {
- ((char *) bin_copy_buffer + bin_copy_buffer_length)[i] =
- ((char *) src)[len - i - 1];
- }
+ return;
+ }
+
+ for (int32_t i = 0; i < len; i++)
+ {
+ ((char *) bin_copy_buffer + bin_copy_buffer_length)[i] =
+ ((char *) src)[len - i - 1];
+ }
bin_copy_buffer_length += len;
#endif
@@ -5322,7 +5328,7 @@ flushBuffer(PGconn *con, int16_t row_len)
* Sends current branch row to buffer
*/
static void
-initBranchBinary(PGconn *con, int64_t curr, int32_t parent)
+initBranchBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
/*
* Each row of branch table has following extra bytes:
@@ -5333,7 +5339,7 @@ initBranchBinary(PGconn *con, int64_t curr, int32_t parent)
flushBuffer(con, max_row_len);
- addColumnCounter(2);
+ addColumnCounter(columnCounter);
addInt32Column(curr + 1);
addInt32Column(0);
@@ -5343,7 +5349,7 @@ initBranchBinary(PGconn *con, int64_t curr, int32_t parent)
* Sends current teller row to buffer
*/
static void
-initTellerBinary(PGconn *con, int64_t curr, int32_t parent)
+initTellerBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
/*
* Each row of tellers table has following extra bytes:
@@ -5354,7 +5360,7 @@ initTellerBinary(PGconn *con, int64_t curr, int32_t parent)
flushBuffer(con, max_row_len);
- addColumnCounter(3);
+ addColumnCounter(columnCounter);
addInt32Column(curr + 1);
addInt32Column(curr / parent + 1);
@@ -5365,7 +5371,7 @@ initTellerBinary(PGconn *con, int64_t curr, int32_t parent)
* Sends current account row to buffer
*/
static void
-initAccountBinary(PGconn *con, int64_t curr, int32_t parent)
+initAccountBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
/*
* Each row of accounts table has following extra bytes:
@@ -5376,7 +5382,7 @@ initAccountBinary(PGconn *con, int64_t curr, int32_t parent)
flushBuffer(con, max_row_len);
- addColumnCounter(3);
+ addColumnCounter(columnCounter);
if (scale <= SCALE_32BIT_THRESHOLD)
addInt32Column(curr + 1);
@@ -5391,8 +5397,9 @@ initAccountBinary(PGconn *con, int64_t curr, int32_t parent)
* Universal wrapper for sending data in binary format
*/
static void
-initPopulateTableBinary(PGconn *con, char *table, char *columns,
- int counter, int64_t base, initRowMethodBin init_row)
+initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
+ int counter, int64_t base, initRowMethodBinary init_row,
+ int columnCounter)
{
int n;
PGresult *res;
@@ -5426,7 +5433,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
for (int64_t i = start; i < start + base; i++)
{
- init_row(con, i, base);
+ init_row(con, i, base, columnCounter);
}
res = PQgetResult(con);
@@ -5460,7 +5467,7 @@ initPopulateTableBinary(PGconn *con, char *table, char *columns,
* Wrapper for binary data load
*/
static void
-initGenerateDataClientSideBinary(PGconn *con)
+initGenerateDataClientSideBinaryFrmt(PGconn *con)
{
fprintf(stderr, "BINARY mode...\n");
@@ -5486,12 +5493,12 @@ initGenerateDataClientSideBinary(PGconn *con)
if (multi_xact)
executeStatement(con, "begin");
- initPopulateTableBinary(con, "pgbench_branches", "bid, bbalance",
- i, nbranches, initBranchBinary);
- initPopulateTableBinary(con, "pgbench_tellers", "tid, bid, tbalance",
- i, ntellers, initTellerBinary);
- initPopulateTableBinary(con, "pgbench_accounts", "aid, bid, abalance",
- i, naccounts, initAccountBinary);
+ initPopulateTableCopyBinary(con, "pgbench_branches", "bid, bbalance",
+ i, nbranches, initBranchBinary, 2);
+ initPopulateTableCopyBinary(con, "pgbench_tellers", "tid, bid, tbalance",
+ i, ntellers, initTellerBinary, 3);
+ initPopulateTableCopyBinary(con, "pgbench_accounts", "aid, bid, abalance",
+ i, naccounts, initAccountBinary, 3);
if (multi_xact)
executeStatement(con, "commit");
@@ -5515,10 +5522,10 @@ initGenerateDataClientSide(PGconn *con)
switch (data_generation_type)
{
case INIT_STEP_GEN_TYPE_COPY_TEXT:
- initGenerateDataClientSideText(con);
+ initGenerateDataClientSideTextFrmt(con);
break;
case INIT_STEP_GEN_TYPE_COPY_BINARY:
- initGenerateDataClientSideBinary(con);
+ initGenerateDataClientSideBinaryFrmt(con);
break;
}
}
--
2.43.0
From d95c53d412ff0e94e162eaf3990233fae007ae3f Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 16 Feb 2026 10:07:13 +0700
Subject: [PATCH v11 20/28] Running indent to fix code formatting
---
src/bin/pgbench/pgbench.c | 111 +++++++++++++++++++++-----------------
1 file changed, 62 insertions(+), 49 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 9f3d7f99a25..8198f1d4c9a 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -159,7 +159,7 @@ typedef struct socket_set
/********************************************************************
* some configurable parameters */
-#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
+#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
#define ALL_INIT_STEPS "dtgMScGUvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
@@ -171,16 +171,24 @@ typedef struct socket_set
#define MAX_ZIPFIAN_PARAM 1000.0 /* maximum parameter for zipfian */
/* server-side methods to generate data */
-#define INIT_STEP_GEN_TYPE_INSERT_SERIES 'G' /* use INSERT .. SELECT generate_series to generate data */
-#define INIT_STEP_GEN_TYPE_INSERT_UNNEST 'U' /* use INSERT .. SELECT unnest to generate data */
+#define INIT_STEP_GEN_TYPE_INSERT_SERIES 'G' /* use INSERT .. SELECT
+ * generate_series to generate
+ * data */
+#define INIT_STEP_GEN_TYPE_INSERT_UNNEST 'U' /* use INSERT .. SELECT unnest
+ * to generate data */
/* client-side methods to generate data */
-#define INIT_STEP_GEN_TYPE_COPY_TEXT 'g' /* use COPY .. FROM STDIN .. TEXT to generate data */
-#define INIT_STEP_GEN_TYPE_COPY_BINARY 'c' /* use COPY .. FROM STDIN .. BINARY to generate data */
+#define INIT_STEP_GEN_TYPE_COPY_TEXT 'g' /* use COPY .. FROM STDIN ..
+ * TEXT to generate data */
+#define INIT_STEP_GEN_TYPE_COPY_BINARY 'c' /* use COPY .. FROM STDIN ..
+ * BINARY to generate data */
/* data init pseudo steps */
-#define INIT_STEP_GEN_TYPE_SINGLE_XACT 'S' /* switch to init data as single transaction */
-#define INIT_STEP_GEN_TYPE_MULTI_XACT 'M' /* switch to init data as multiple transactions */
+#define INIT_STEP_GEN_TYPE_SINGLE_XACT 'S' /* switch to init data as
+ * single transaction */
+#define INIT_STEP_GEN_TYPE_MULTI_XACT 'M' /* switch to init data as
+ * multiple transactions */
-static bool multi_xact = false; /* init data type (as single or multiple transactions) */
+static bool multi_xact = false; /* init data type (as single or multiple
+ * transactions) */
static int nxacts = 0; /* number of transactions per client */
static int duration = 0; /* duration in seconds */
static int64 end_time = 0; /* when to stop in micro seconds, under -T */
@@ -194,14 +202,15 @@ static int scale = 1;
/*
* mode of data generation to use
*/
-static char data_generation_type = INIT_STEP_GEN_TYPE_COPY_TEXT;
+static char data_generation_type = INIT_STEP_GEN_TYPE_COPY_TEXT;
/*
* COPY FROM BINARY execution buffer
*/
-#define BIN_COPY_BUF_SIZE 102400 /* maximum buffer size for COPY FROM BINARY */
-static char *bin_copy_buffer = NULL; /* buffer for COPY FROM BINARY */
-static int32_t bin_copy_buffer_length = 0; /* current buffer size */
+#define BIN_COPY_BUF_SIZE 102400 /* maximum buffer size for COPY FROM
+ * BINARY */
+static char *bin_copy_buffer = NULL; /* buffer for COPY FROM BINARY */
+static int32_t bin_copy_buffer_length = 0; /* current buffer size */
/*
* fillfactor. for example, fillfactor = 90 will use only 90 percent
@@ -872,8 +881,8 @@ static int wait_on_socket_set(socket_set *sa, int64 usecs);
static bool socket_has_input(socket_set *sa, int fd, int idx);
/* callback used to build rows for COPY during data loading */
-typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
-typedef void (*initRowMethodBinary) (PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter);
+typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
+typedef void (*initRowMethodBinary) (PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter);
/* callback functions for our flex lexer */
static const PsqlScanCallbacks pgbench_callbacks = {
@@ -5200,7 +5209,7 @@ initGenerateDataClientSideTextFrmt(PGconn *con)
* already exist
*/
initPopulateTableCopyText(con, "pgbench_branches", nbranches, initBranch);
- initPopulateTableCopyText(con, "pgbench_tellers", ntellers, initTeller);
+ initPopulateTableCopyText(con, "pgbench_tellers", ntellers, initTeller);
initPopulateTableCopyText(con, "pgbench_accounts", naccounts, initAccount);
executeStatement(con, "commit");
@@ -5260,8 +5269,9 @@ addColumnCounter(int16_t n)
static void
addInt32Column(int32_t value)
{
- int32_t data = value;
- int32_t size = sizeof(data);
+ int32_t data = value;
+ int32_t size = sizeof(data);
+
bufferData((void *) &size, sizeof(size));
bufferData((void *) &data, sizeof(data));
}
@@ -5272,8 +5282,9 @@ addInt32Column(int32_t value)
static void
addInt64Column(int64_t value)
{
- int64_t data = value;
- int32_t size = sizeof(data);
+ int64_t data = value;
+ int32_t size = sizeof(data);
+
bufferData((void *) &size, sizeof(size));
bufferData((void *) &data, sizeof(data));
}
@@ -5284,9 +5295,9 @@ addInt64Column(int64_t value)
static void
sendBinaryCopyHeader(PGconn *con)
{
- char header[] = {'P','G','C','O','P','Y','\n','\377','\r','\n','\0',
- '\0','\0','\0','\0',
- '\0','\0','\0','\0' };
+ char header[] = {'P', 'G', 'C', 'O', 'P', 'Y', '\n', '\377', '\r', '\n', '\0',
+ '\0', '\0', '\0', '\0',
+ '\0', '\0', '\0', '\0'};
PQputCopyData(con, header, 19);
}
@@ -5297,7 +5308,7 @@ sendBinaryCopyHeader(PGconn *con)
static void
sendBinaryCopyTrailer(PGconn *con)
{
- static char trailer[] = { 0xFF, 0xFF };
+ static char trailer[] = {0xFF, 0xFF};
PQputCopyData(con, trailer, 2);
}
@@ -5308,7 +5319,7 @@ sendBinaryCopyTrailer(PGconn *con)
static void
flushBuffer(PGconn *con, int16_t row_len)
{
- PGresult *res;
+ PGresult *res;
if (bin_copy_buffer_length + row_len > BIN_COPY_BUF_SIZE)
{
@@ -5330,12 +5341,13 @@ flushBuffer(PGconn *con, int16_t row_len)
static void
initBranchBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
- /*
+ /*---
* Each row of branch table has following extra bytes:
* - 2 bytes for number of columns
* - 4 bytes as length for each of table's 3 columns
+ *---
*/
- int16_t max_row_len = 35 + 2 + 4*3; /* max row size is 35 */
+ int16_t max_row_len = 35 + 2 + 4 * 3; /* max row size is 35 */
flushBuffer(con, max_row_len);
@@ -5351,12 +5363,13 @@ initBranchBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter
static void
initTellerBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
- /*
+ /*---
* Each row of tellers table has following extra bytes:
* - 2 bytes for number of columns
* - 4 bytes as length for each of table's 4 columns
+ *---
*/
- int16_t max_row_len = 40 + 2 + 4*4; /* max row size is 40 */
+ int16_t max_row_len = 40 + 2 + 4 * 4; /* max row size is 40 */
flushBuffer(con, max_row_len);
@@ -5373,12 +5386,13 @@ initTellerBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter
static void
initAccountBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
{
- /*
+ /*---
* Each row of accounts table has following extra bytes:
* - 2 bytes for number of columns
* - 4 bytes as length for each of table's 4 columns
*/
- int16_t max_row_len = 250 + 2 + 4*4; /* max row size is 250 for int64 */
+ int16_t max_row_len = 250 + 2 + 4 * 4; /* max row size is 250 for
+ * int64 */
flushBuffer(con, max_row_len);
@@ -5401,11 +5415,11 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
int counter, int64_t base, initRowMethodBinary init_row,
int columnCounter)
{
- int n;
- PGresult *res;
- char copy_statement[256];
- const char *copy_statement_fmt = "copy %s (%s) from stdin (format binary)";
- int64_t start = base * counter;
+ int n;
+ PGresult *res;
+ char copy_statement[256];
+ const char *copy_statement_fmt = "copy %s (%s) from stdin (format binary)";
+ int64_t start = base * counter;
bin_copy_buffer_length = 0;
@@ -5450,7 +5464,7 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
res = PQgetResult(con);
if (PQresultStatus(res) == PGRES_COPY_IN)
{
- if (PQputCopyEnd(con, NULL) == 1) /* success */
+ if (PQputCopyEnd(con, NULL) == 1) /* success */
{
PQclear(res);
res = PQgetResult(con);
@@ -5476,9 +5490,8 @@ initGenerateDataClientSideBinaryFrmt(PGconn *con)
bin_copy_buffer_length = 0;
/*
- * we do all of this in multiple transactions
- * to minimize load on DB server and perhaps
- * in future allow load in parallel sessions
+ * we do all of this in multiple transactions to minimize load on DB
+ * server and perhaps in future allow load in parallel sessions
*/
executeStatement(con, "begin");
@@ -5495,8 +5508,8 @@ initGenerateDataClientSideBinaryFrmt(PGconn *con)
initPopulateTableCopyBinary(con, "pgbench_branches", "bid, bbalance",
i, nbranches, initBranchBinary, 2);
- initPopulateTableCopyBinary(con, "pgbench_tellers", "tid, bid, tbalance",
- i, ntellers, initTellerBinary, 3);
+ initPopulateTableCopyBinary(con, "pgbench_tellers", "tid, bid, tbalance",
+ i, ntellers, initTellerBinary, 3);
initPopulateTableCopyBinary(con, "pgbench_accounts", "aid, bid, abalance",
i, naccounts, initAccountBinary, 3);
@@ -5570,10 +5583,10 @@ generateDataInsertSeries(PGconn *con)
printfPQExpBuffer(&sql,
"insert into pgbench_accounts(aid, bid, abalance, "
- "filler) "
+ "filler) "
"select aid + 1, aid / %d + 1, 0, '' "
"from generate_series(" INT64_FORMAT ", "
- INT64_FORMAT ") as aid",
+ INT64_FORMAT ") as aid",
naccounts, (int64) i * naccounts,
(int64) (i + 1) * naccounts - 1);
executeStatement(con, sql.data);
@@ -5622,7 +5635,7 @@ generateDataInsertUnnest(PGconn *con)
printfPQExpBuffer(&sql,
"insert into pgbench_tellers(tid, bid, tbalance) "
"select unnest(array_agg(s.i order by s.i)) as tid, "
- "%d as bid, 0 as tbalance "
+ "%d as bid, 0 as tbalance "
"from generate_series(%d, %d) as s(i)",
s + 1, s * ntellers + 1, (s + 1) * ntellers);
executeStatement(con, sql.data);
@@ -5630,13 +5643,13 @@ generateDataInsertUnnest(PGconn *con)
printfPQExpBuffer(&sql,
"with data as ("
" select generate_series(" INT64_FORMAT ", "
- INT64_FORMAT ") as i) "
+ INT64_FORMAT ") as i) "
"insert into pgbench_accounts(aid, bid, "
- "abalance, filler) "
+ "abalance, filler) "
"select unnest(aid), unnest(bid), 0 as abalance, "
- "'' as filler "
+ "'' as filler "
"from (select array_agg(i+1) aid, "
- "array_agg(i/%d + 1) bid from data)",
+ "array_agg(i/%d + 1) bid from data)",
(int64) s * naccounts + 1,
(int64) (s + 1) * naccounts, naccounts);
executeStatement(con, sql.data);
@@ -5758,7 +5771,7 @@ initCreateFKeys(PGconn *con)
static void
checkInitSteps(const char *initialize_steps)
{
- char data_init_type = 0;
+ char data_init_type = 0;
if (initialize_steps[0] == '\0')
pg_fatal("no initialization steps specified");
--
2.43.0
From 270d2bf14b6873ff66fc37ca0a5dccc055dab1f7 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 16 Feb 2026 14:57:18 +0700
Subject: [PATCH v11 21/28] Removing debugging filler value
---
src/bin/pgbench/pgbench.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 8198f1d4c9a..710b2a5b70d 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -4954,8 +4954,8 @@ initCreateTables(PGconn *con)
static const struct ddlinfo DDLs[] = {
{
"pgbench_history",
- "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22) default '?'",
- "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22) default '?'",
+ "tid int,bid int,aid int,delta int,mtime timestamp,filler char(22) default ''",
+ "tid int,bid int,aid bigint,delta int,mtime timestamp,filler char(22) default ''",
0
},
{
@@ -4966,8 +4966,8 @@ initCreateTables(PGconn *con)
},
{
"pgbench_accounts",
- "aid int not null,bid int,abalance int,filler char(84) default '?'",
- "aid bigint not null,bid int,abalance int,filler char(84) default '?'",
+ "aid int not null,bid int,abalance int,filler char(84) default ''",
+ "aid bigint not null,bid int,abalance int,filler char(84) default ''",
1
},
{
--
2.43.0
From 86a111c63e7006d4801ce7bb594a5b77d87b42b0 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 23 Feb 2026 16:15:06 +0700
Subject: [PATCH v11 22/28] Moving out logic showing data load progress into
own procedure
---
src/bin/pgbench/pgbench.c | 142 ++++++++++++++++++++------------------
1 file changed, 75 insertions(+), 67 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 710b2a5b70d..e3e0e073792 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -487,6 +487,9 @@ typedef struct StatsData
*/
static pg_time_usec_t epoch_shift;
+/* used to track elapsed time and estimate of the remaining time of data load */
+static pg_time_usec_t data_load_start;
+
/*
* Error status for errors during script execution.
*/
@@ -5062,27 +5065,82 @@ initAccount(PQExpBufferData *sql, int64 curr)
curr + 1, curr / naccounts + 1);
}
+static void
+showPopulateTableCopyProgress(const char *table, int64 current, int64 total)
+{
+ static int chars = 0;
+ static int prev_chars = 0;
+ static int log_interval = 1;
+
+ /* Stay on the same line if reporting to a terminal */
+ char eol = isatty(fileno(stderr)) ? '\r' : '\n';
+
+ double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - data_load_start);
+ double remaining_sec = ((double) total - current) * elapsed_sec / current;
+
+ /*
+ * If we want to stick with the original logging, print a message each
+ * 100k inserted rows.
+ */
+ if ((!use_quiet) && (current % 100000 == 0))
+ {
+ chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)",
+ current, total,
+ (int) ((current * 100) / total),
+ table, elapsed_sec, remaining_sec);
+
+ /*
+ * If the previous progress message is longer than the current one,
+ * add spaces to the current line to fully overwrite any remaining
+ * characters from the previous message.
+ */
+ if (prev_chars > chars)
+ fprintf(stderr, "%*c", prev_chars - chars, ' ');
+ fputc(eol, stderr);
+ prev_chars = chars;
+ }
+ /* let's not call the timing for each row, but only each 100 rows */
+ else if (use_quiet && (current % 100 == 0))
+ {
+ /* have we reached the next interval (or end)? */
+ if ((current == total) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
+ {
+ chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)",
+ current, total,
+ (int) ((current * 100) / total),
+ table, elapsed_sec, remaining_sec);
+
+ /*
+ * If the previous progress message is longer than the current
+ * one, add spaces to the current line to fully overwrite any
+ * remaining characters from the previous message.
+ */
+ if (prev_chars > chars)
+ fprintf(stderr, "%*c", prev_chars - chars, ' ');
+ fputc(eol, stderr);
+ prev_chars = chars;
+
+ /* skip to the next interval */
+ log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
+ }
+ }
+
+ if (current == total && chars != 0 && eol != '\n')
+ fprintf(stderr, "%*c\r", chars, ' '); /* Clear the current line */
+}
+
static void
initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
initRowMethod init_row)
{
int n;
int64 k;
- int chars = 0;
- int prev_chars = 0;
PGresult *res;
PQExpBufferData sql;
char copy_statement[256];
const char *copy_statement_fmt = "copy %s from stdin";
int64 total = base * scale;
- /* used to track elapsed time and estimate of the remaining time */
- pg_time_usec_t start;
- int log_interval = 1;
-
- /* Stay on the same line if reporting to a terminal */
- char eol = isatty(fileno(stderr)) ? '\r' : '\n';
-
initPQExpBuffer(&sql);
/* Use COPY with FREEZE on v14 and later for all ordinary tables */
@@ -5103,12 +5161,8 @@ initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
PQclear(res);
- start = pg_time_now();
-
for (k = 0; k < total; k++)
{
- int64 j = k + 1;
-
init_row(&sql, k);
if (PQputline(con, sql.data))
pg_fatal("PQputline failed");
@@ -5116,62 +5170,9 @@ initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
if (CancelRequested)
break;
- /*
- * If we want to stick with the original logging, print a message each
- * 100k inserted rows.
- */
- if ((!use_quiet) && (j % 100000 == 0))
- {
- double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
- double remaining_sec = ((double) total - j) * elapsed_sec / j;
-
- chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)",
- j, total,
- (int) ((j * 100) / total),
- table, elapsed_sec, remaining_sec);
-
- /*
- * If the previous progress message is longer than the current
- * one, add spaces to the current line to fully overwrite any
- * remaining characters from the previous message.
- */
- if (prev_chars > chars)
- fprintf(stderr, "%*c", prev_chars - chars, ' ');
- fputc(eol, stderr);
- prev_chars = chars;
- }
- /* let's not call the timing for each row, but only each 100 rows */
- else if (use_quiet && (j % 100 == 0))
- {
- double elapsed_sec = PG_TIME_GET_DOUBLE(pg_time_now() - start);
- double remaining_sec = ((double) total - j) * elapsed_sec / j;
-
- /* have we reached the next interval (or end)? */
- if ((j == total) || (elapsed_sec >= log_interval * LOG_STEP_SECONDS))
- {
- chars = fprintf(stderr, INT64_FORMAT " of " INT64_FORMAT " tuples (%d%%) of %s done (elapsed %.2f s, remaining %.2f s)",
- j, total,
- (int) ((j * 100) / total),
- table, elapsed_sec, remaining_sec);
-
- /*
- * If the previous progress message is longer than the current
- * one, add spaces to the current line to fully overwrite any
- * remaining characters from the previous message.
- */
- if (prev_chars > chars)
- fprintf(stderr, "%*c", prev_chars - chars, ' ');
- fputc(eol, stderr);
- prev_chars = chars;
-
- /* skip to the next interval */
- log_interval = (int) ceil(elapsed_sec / LOG_STEP_SECONDS);
- }
- }
+ showPopulateTableCopyProgress(table, k + 1, total);
}
- if (chars != 0 && eol != '\n')
- fprintf(stderr, "%*c\r", chars, ' '); /* Clear the current line */
if (PQputline(con, "\\.\n"))
pg_fatal("very last PQputline failed");
@@ -5448,6 +5449,11 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
for (int64_t i = start; i < start + base; i++)
{
init_row(con, i, base, columnCounter);
+
+ if (CancelRequested)
+ break;
+
+ showPopulateTableCopyProgress(table, i, base * scale);
}
res = PQgetResult(con);
@@ -5532,6 +5538,8 @@ initGenerateDataClientSide(PGconn *con)
fprintf(stderr, "generating data (client-side as %s transaction%s) in ",
multi_xact ? "multiple" : "single", multi_xact ? "s" : "");
+ data_load_start = pg_time_now();
+
switch (data_generation_type)
{
case INIT_STEP_GEN_TYPE_COPY_TEXT:
--
2.43.0
From afe3f69b23190f375b57d5cf0a3eae2cd8e40caa Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Mon, 23 Feb 2026 17:07:16 +0700
Subject: [PATCH v11 23/28] Adding multi-transaction mode to client-side data
generation
---
src/bin/pgbench/pgbench.c | 52 ++++++++++++--------
src/bin/pgbench/t/001_pgbench_with_server.pl | 47 +++++++++---------
2 files changed, 56 insertions(+), 43 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index e3e0e073792..1b73da904ad 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5130,22 +5130,22 @@ showPopulateTableCopyProgress(const char *table, int64 current, int64 total)
}
static void
-initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
+initPopulateTableCopyText(PGconn *con, const char *table, int counter, int64 base,
initRowMethod init_row)
{
int n;
- int64 k;
PGresult *res;
PQExpBufferData sql;
char copy_statement[256];
const char *copy_statement_fmt = "copy %s from stdin";
- int64 total = base * scale;
+ int64 start = base * counter;
initPQExpBuffer(&sql);
/* Use COPY with FREEZE on v14 and later for all ordinary tables */
if ((PQserverVersion(con) >= 140000) &&
- get_table_relkind(con, table) == RELKIND_RELATION)
+ get_table_relkind(con, table) == RELKIND_RELATION &&
+ !multi_xact)
copy_statement_fmt = "copy %s from stdin with (freeze on)";
@@ -5161,16 +5161,16 @@ initPopulateTableCopyText(PGconn *con, const char *table, int64 base,
pg_fatal("unexpected copy in result: %s", PQerrorMessage(con));
PQclear(res);
- for (k = 0; k < total; k++)
+ for (int64_t i = start; i < start + base; i++)
{
- init_row(&sql, k);
+ init_row(&sql, i);
if (PQputline(con, sql.data))
pg_fatal("PQputline failed");
if (CancelRequested)
break;
- showPopulateTableCopyProgress(table, k + 1, total);
+ showPopulateTableCopyProgress(table, i, base * scale);
}
@@ -5193,9 +5193,6 @@ initGenerateDataClientSideTextFrmt(PGconn *con)
{
fprintf(stderr, "TEXT mode...\n");
- if (multi_xact)
- fprintf(stderr, "WARNING! Multiple transactions are not supported in this mode\n");
-
/*
* we do all of this in one transaction to enable the backend's
* data-loading optimizations
@@ -5205,15 +5202,28 @@ initGenerateDataClientSideTextFrmt(PGconn *con)
/* truncate away any old data */
initTruncateTables(con);
- /*
- * fill branches, tellers, accounts in that order in case foreign keys
- * already exist
- */
- initPopulateTableCopyText(con, "pgbench_branches", nbranches, initBranch);
- initPopulateTableCopyText(con, "pgbench_tellers", ntellers, initTeller);
- initPopulateTableCopyText(con, "pgbench_accounts", naccounts, initAccount);
+ if (multi_xact)
+ executeStatement(con, "commit");
+
+ for (int i = 0; i < scale; i++)
+ {
+ if (multi_xact)
+ executeStatement(con, "begin");
+
+ /*
+ * fill branches, tellers, accounts in that order in case foreign keys
+ * already exist
+ */
+ initPopulateTableCopyText(con, "pgbench_branches", i, nbranches, initBranch);
+ initPopulateTableCopyText(con, "pgbench_tellers", i, ntellers, initTeller);
+ initPopulateTableCopyText(con, "pgbench_accounts", i, naccounts, initAccount);
- executeStatement(con, "commit");
+ if (multi_xact)
+ executeStatement(con, "commit");
+ }
+
+ if (!multi_xact)
+ executeStatement(con, "commit");
}
@@ -5426,9 +5436,9 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
/* Use COPY with FREEZE on v14 and later for all ordinary tables */
if ((PQserverVersion(con) >= 140000) &&
- get_table_relkind(con, table) == RELKIND_RELATION)
- if (!multi_xact)
- copy_statement_fmt = "copy %s (%s) from stdin with (format binary, freeze on)";
+ get_table_relkind(con, table) == RELKIND_RELATION &&
+ !multi_xact)
+ copy_statement_fmt = "copy %s (%s) from stdin with (format binary, freeze on)";
n = pg_snprintf(copy_statement, sizeof(copy_statement), copy_statement_fmt, table, columns);
if (n >= sizeof(copy_statement))
diff --git a/src/bin/pgbench/t/001_pgbench_with_server.pl b/src/bin/pgbench/t/001_pgbench_with_server.pl
index acb1e31d3e8..46569b0eb37 100644
--- a/src/bin/pgbench/t/001_pgbench_with_server.pl
+++ b/src/bin/pgbench/t/001_pgbench_with_server.pl
@@ -189,7 +189,7 @@ $node->pgbench(
check_data_state($node, 'server-side (unnest)');
-# Test server-side generation with COPY TEXT
+# Test client-side generation with COPY TEXT
$node->pgbench(
'--initialize --init-steps=dtg',
0,
@@ -200,45 +200,48 @@ $node->pgbench(
qr{generating data \(client-side as single transaction},
qr{done in \d+\.\d\d s }
],
- 'pgbench --init-steps client-side TEXT');
+ 'pgbench --init-steps client-side TEXT (single XACT #1)');
-# Check data state, after server-side data generation.
+# Check data state, after client-side data generation.
check_data_state($node, 'client-side (text)');
$node->pgbench(
- '--initialize --init-steps=dtMg',
+ '--initialize --init-steps=dtSg',
0,
[qr{^$}],
[
qr{dropping old tables},
qr{creating tables},
- qr{generating data \(client-side as multiple transactions},
+ qr{generating data \(client-side as single transaction},
+ qr{\d of \d+ tuples \(\d%\) of pgbench_branches done},
+ qr{\d of \d+ tuples \(\d%\) of pgbench_tellers done},
+ qr{\d of \d+ tuples \(\d%\) of pgbench_accounts done},
qr{done in \d+\.\d\d s }
],
- 'pgbench --init-steps client-side TEXT');
+ 'pgbench --init-steps client-side TEXT (single XACT #2)');
-# Check data state, after server-side data generation.
+# Check data state, after client-side data generation.
check_data_state($node, 'client-side (text)');
-
-# Test server-side generation with COPY BINARY
$node->pgbench(
- '--initialize --init-steps=dtc',
+ '--initialize --init-steps=dtMg',
0,
[qr{^$}],
[
qr{dropping old tables},
qr{creating tables},
- qr{generating data \(client-side as single transaction},
+ qr{generating data \(client-side as multiple transactions},
qr{done in \d+\.\d\d s }
],
- 'pgbench --init-steps client-side BINARY');
+ 'pgbench --init-steps client-side TEXT (multiple XACTs)');
+
+# Check data state, after client-side data generation.
+check_data_state($node, 'client-side (text)');
-# Check data state, after server-side data generation.
-check_data_state($node, 'client-side (binary)');
+# Test client-side generation with COPY BINARY
$node->pgbench(
- '--initialize --init-steps=dtSc',
+ '--initialize --init-steps=dtc',
0,
[qr{^$}],
[
@@ -247,24 +250,24 @@ $node->pgbench(
qr{generating data \(client-side as single transaction},
qr{done in \d+\.\d\d s }
],
- 'pgbench --init-steps client-side BINARY');
+ 'pgbench --init-steps client-side BINARY (single XACT #1)');
-# Check data state, after server-side data generation.
+# Check data state, after client-side data generation.
check_data_state($node, 'client-side (binary)');
$node->pgbench(
- '--initialize --init-steps=dtMc',
+ '--initialize --init-steps=dtSc',
0,
[qr{^$}],
[
qr{dropping old tables},
qr{creating tables},
- qr{generating data \(client-side as multiple transactions},
+ qr{generating data \(client-side as single transaction},
qr{done in \d+\.\d\d s }
],
- 'pgbench --init-steps client-side BINARY');
+ 'pgbench --init-steps client-side BINARY (single XACT #2)');
-# Check data state, after server-side data generation.
+# Check data state, after client-side data generation.
check_data_state($node, 'client-side (binary)');
$node->pgbench(
@@ -279,7 +282,7 @@ $node->pgbench(
],
'pgbench --init-steps client-side BINARY');
-# Check data state, after server-side data generation.
+# Check data state, after client-side data generation.
check_data_state($node, 'client-side (binary)');
--
2.43.0
From d9c61efdd853c999b92061ad4f88dd012265dbba Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 1 Mar 2026 18:24:18 +0700
Subject: [PATCH v11 24/28] Removing double call to checkInitSteps
---
src/bin/pgbench/pgbench.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 1b73da904ad..7d3a1583152 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5816,9 +5816,9 @@ checkInitSteps(const char *initialize_steps)
}
if (data_init_type == 0)
- pg_log_error("WARNING! No data generation type is provided");
+ pg_log_warning("No data generation type is provided");
if (data_init_type > 1)
- pg_log_error("WARNING! More than one type of data initialization is requested");
+ pg_log_warning("More than one type of data initialization is requested");
}
/*
@@ -7449,7 +7449,6 @@ main(int argc, char **argv)
case 'I':
pg_free(initialize_steps);
initialize_steps = pg_strdup(optarg);
- checkInitSteps(initialize_steps);
initialization_option_set = true;
break;
case 'j': /* jobs */
--
2.43.0
From d593a2063a997c86de4b9350b97d4ad6527bddef Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 1 Mar 2026 18:45:25 +0700
Subject: [PATCH v11 25/28] Fixed 2 small bugs
1. Overwrite of init mode by last mode in checkInitSteps
2. Bledding of init progress from previous mode under new one (eg, '-IggGG')
---
src/bin/pgbench/pgbench.c | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 7d3a1583152..daf33d73814 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -5125,8 +5125,11 @@ showPopulateTableCopyProgress(const char *table, int64 current, int64 total)
}
}
- if (current == total && chars != 0 && eol != '\n')
- fprintf(stderr, "%*c\r", chars, ' '); /* Clear the current line */
+ if (current + 1 == total && chars != 0)
+ {
+ fprintf(stderr, "%*c", chars, ' '); /* Clear the current line */
+ fputc(eol, stderr);
+ }
}
static void
@@ -5810,7 +5813,6 @@ checkInitSteps(const char *initialize_steps)
case INIT_STEP_GEN_TYPE_INSERT_SERIES:
case INIT_STEP_GEN_TYPE_INSERT_UNNEST:
data_init_type++;
- data_generation_type = *step;
break;
}
}
@@ -5859,11 +5861,13 @@ runInitSteps(const char *initialize_steps)
case INIT_STEP_GEN_TYPE_COPY_TEXT:
case INIT_STEP_GEN_TYPE_COPY_BINARY:
op = "client-side generate";
+ data_generation_type = *step;
initGenerateDataClientSide(con);
break;
case INIT_STEP_GEN_TYPE_INSERT_SERIES:
case INIT_STEP_GEN_TYPE_INSERT_UNNEST:
op = "server-side generate";
+ data_generation_type = *step;
initGenerateDataServerSide(con);
break;
case INIT_STEP_GEN_TYPE_SINGLE_XACT:
--
2.43.0
From 265d4b017529c9bdc21ecdbf7fe4794789260545 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Sun, 1 Mar 2026 19:06:45 +0700
Subject: [PATCH v11 26/28] Added more tests
---
src/bin/pgbench/t/001_pgbench_with_server.pl | 98 ++++++++++++++++++++
1 file changed, 98 insertions(+)
diff --git a/src/bin/pgbench/t/001_pgbench_with_server.pl b/src/bin/pgbench/t/001_pgbench_with_server.pl
index 46569b0eb37..73d2d754273 100644
--- a/src/bin/pgbench/t/001_pgbench_with_server.pl
+++ b/src/bin/pgbench/t/001_pgbench_with_server.pl
@@ -172,6 +172,53 @@ $node->pgbench(
check_data_state($node, 'server-side');
+# Test server-side generation with generate_series
+$node->pgbench(
+ '--initialize --init-steps=dtG',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side as single transaction\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side generate_series');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (generate_series)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtSG',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side as single transaction\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side generate_series');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (generate_series single XACT)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMG',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side as multiple transactions\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side generate_series');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (generate_series multiple XACTs)');
+
+
# Test server-side generation with UNNEST
$node->pgbench(
'--initialize --init-steps=dtU',
@@ -188,6 +235,36 @@ $node->pgbench(
# Check data state, after server-side data generation.
check_data_state($node, 'server-side (unnest)');
+$node->pgbench(
+ '--initialize --init-steps=dtSU',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side as single transaction\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side UNNEST');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (unnest)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMU',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(server-side as multiple transactions\)},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps server-side UNNEST');
+
+# Check data state, after server-side data generation.
+check_data_state($node, 'server-side (unnest)');
+
# Test client-side generation with COPY TEXT
$node->pgbench(
@@ -286,6 +363,27 @@ $node->pgbench(
check_data_state($node, 'client-side (binary)');
+# Check data state, after different modes of client-side data generation.
+check_data_state($node, 'client-side (binary)');
+
+$node->pgbench(
+ '--initialize --init-steps=dtMccSc',
+ 0,
+ [qr{^$}],
+ [
+ qr{dropping old tables},
+ qr{creating tables},
+ qr{generating data \(client-side as multiple transactions},
+ qr{generating data \(client-side as multiple transactions},
+ qr{generating data \(client-side as single transaction},
+ qr{done in \d+\.\d\d s }
+ ],
+ 'pgbench --init-steps client-side BINARY (multiple XACT modes)');
+
+# Check data state, after client-side data generation.
+check_data_state($node, 'client-side (binary different XACT modes in list of --init-steps)');
+
+
# Run all builtin scripts, for a few transactions each
$node->pgbench(
'--transactions=5 -Dfoo=bla --client=2 --protocol=simple --builtin=t'
--
2.43.0
From 3f569dfba37c8722fb5e2b99f3e001a93a959511 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Tue, 3 Mar 2026 10:49:54 +0700
Subject: [PATCH v11 27/28] Small changes due to review of patch v9
---
src/bin/pgbench/pgbench.c | 26 +++++++++++++++-----------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index daf33d73814..81576d8f7ca 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -160,7 +160,7 @@ typedef struct socket_set
/********************************************************************
* some configurable parameters */
#define DEFAULT_INIT_STEPS "dtgvp" /* default -I setting */
-#define ALL_INIT_STEPS "dtgMScGUvpf" /* all possible steps */
+#define ALL_INIT_STEPS "dtMScgGUvpf" /* all possible steps */
#define LOG_STEP_SECONDS 5 /* seconds between log messages */
#define DEFAULT_NXACTS 10 /* default nxacts */
@@ -5232,6 +5232,8 @@ initGenerateDataClientSideTextFrmt(PGconn *con)
/*
* Save char data to buffer
+ * Kept as separate proc for possible addition of something
+ * like addCharColumn in future
*/
static void
bufferCharData(char *src, int32_t len)
@@ -5249,22 +5251,24 @@ bufferCharData(char *src, int32_t len)
static void
bufferData(void *src, int32_t len)
{
+ Assert(bin_copy_buffer_length + len <= BIN_COPY_BUF_SIZE);
+
#ifdef __sparc__
bufferCharData(src, len);
#else
+
if (len == 1)
- {
bufferCharData(src, len);
- return;
- }
-
- for (int32_t i = 0; i < len; i++)
+ else
{
- ((char *) bin_copy_buffer + bin_copy_buffer_length)[i] =
- ((char *) src)[len - i - 1];
- }
+ for (int32_t i = 0; i < len; i++)
+ {
+ ((char *) bin_copy_buffer + bin_copy_buffer_length)[i] =
+ ((char *) src)[len - i - 1];
+ }
- bin_copy_buffer_length += len;
+ bin_copy_buffer_length += len;
+ }
#endif
}
@@ -5309,7 +5313,7 @@ addInt64Column(int64_t value)
static void
sendBinaryCopyHeader(PGconn *con)
{
- char header[] = {'P', 'G', 'C', 'O', 'P', 'Y', '\n', '\377', '\r', '\n', '\0',
+ static char header[] = {'P', 'G', 'C', 'O', 'P', 'Y', '\n', '\377', '\r', '\n', '\0',
'\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0'};
--
2.43.0
From 701c438f4adcea84784a1dd478e60902f9083eb7 Mon Sep 17 00:00:00 2001
From: Boris Mironov <[email protected]>
Date: Tue, 3 Mar 2026 18:01:56 +0700
Subject: [PATCH v11 28/28] Rewrote comments and formulas calculating
max_row_len used by COPY BINARY
---
src/bin/pgbench/pgbench.c | 93 +++++++++++++++++++++++++++------------
1 file changed, 65 insertions(+), 28 deletions(-)
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 81576d8f7ca..971a1d3a3d8 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -885,7 +885,7 @@ static bool socket_has_input(socket_set *sa, int fd, int idx);
/* callback used to build rows for COPY during data loading */
typedef void (*initRowMethod) (PQExpBufferData *sql, int64 curr);
-typedef void (*initRowMethodBinary) (PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter);
+typedef void (*initRowMethodBinary) (PGconn *con, int64_t curr, int32_t parent);
/* callback functions for our flex lexer */
static const PsqlScanCallbacks pgbench_callbacks = {
@@ -5127,7 +5127,7 @@ showPopulateTableCopyProgress(const char *table, int64 current, int64 total)
if (current + 1 == total && chars != 0)
{
- fprintf(stderr, "%*c", chars, ' '); /* Clear the current line */
+ fprintf(stderr, "%*c", chars, ' '); /* Clear the current line */
fputc(eol, stderr);
}
}
@@ -5313,7 +5313,7 @@ addInt64Column(int64_t value)
static void
sendBinaryCopyHeader(PGconn *con)
{
- static char header[] = {'P', 'G', 'C', 'O', 'P', 'Y', '\n', '\377', '\r', '\n', '\0',
+ static char header[] = {'P', 'G', 'C', 'O', 'P', 'Y', '\n', '\377', '\r', '\n', '\0',
'\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0'};
@@ -5342,6 +5342,9 @@ flushBuffer(PGconn *con, int16_t row_len)
if (bin_copy_buffer_length + row_len > BIN_COPY_BUF_SIZE)
{
res = PQgetResult(con);
+
+ Assert(bin_copy_buffer_length <= BIN_COPY_BUF_SIZE);
+
/* flush current buffer */
if (PQresultStatus(res) == PGRES_COPY_IN)
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
@@ -5357,64 +5360,95 @@ flushBuffer(PGconn *con, int16_t row_len)
* Sends current branch row to buffer
*/
static void
-initBranchBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
+initBranchBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*---
- * Each row of branch table has following extra bytes:
- * - 2 bytes for number of columns
- * - 4 bytes as length for each of table's 3 columns
+ * Check documentation about COPY command:
+ * https://www.postgresql.org/docs/current/sql-copy.html
+ *
+ * Each row of branches table is sent as:
+ * - 2 bytes for number of columns in tuple or sizeof(int16_t)
+ * - then 4 bytes or sizeof(int32_t) in front of each field with length of the field
+ *
+ * - branches table has following columns:
+ * - 4 bytes for bid column or sizeof(int32_t)
+ * - 4 bytes for bbalance column or sizeof(int32_t)
+ * - 88 bytes for filler column (optional since no requirement for row length)
*---
*/
- int16_t max_row_len = 35 + 2 + 4 * 3; /* max row size is 35 */
+ /* following is our max intent at the moment */
+ int16_t max_row_len = 2 + (4 + 4) + (4 + 4) + (4 + 88);
flushBuffer(con, max_row_len);
- addColumnCounter(columnCounter);
+ addColumnCounter(2);
addInt32Column(curr + 1);
addInt32Column(0);
+ /* we don't send filler column here to minimize network traffic and WAL */
}
/*
* Sends current teller row to buffer
*/
static void
-initTellerBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
+initTellerBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*---
- * Each row of tellers table has following extra bytes:
- * - 2 bytes for number of columns
- * - 4 bytes as length for each of table's 4 columns
+ * Check documentation about COPY command:
+ * https://www.postgresql.org/docs/current/sql-copy.html
+ *
+ * Each row of tellers table is sent as:
+ * - 2 bytes for number of columns in tuple or sizeof(int16_t)
+ * - then 4 bytes or sizeof(int32_t) in front of each field with length of the field
+ *
+ * - tellers table has following columns:
+ * - 4 bytes for tid column or sizeof(int32_t)
+ * - 4 bytes for bid column or sizeof(int32_t)
+ * - 4 bytes for tbalance column or sizeof(int32_t)
+ * - 84 bytes for filler column (optional since no requirement for row length)
*---
*/
- int16_t max_row_len = 40 + 2 + 4 * 4; /* max row size is 40 */
+ /* following is our max intent at the moment */
+ int16_t max_row_len = 2 + (4 + 4) + (4 + 4) + (4 + 4) + (4 + 84);
flushBuffer(con, max_row_len);
- addColumnCounter(columnCounter);
+ addColumnCounter(3);
addInt32Column(curr + 1);
addInt32Column(curr / parent + 1);
addInt32Column(0);
+ /* we don't send filler column here to minimize network traffic and WAL */
}
/*
* Sends current account row to buffer
*/
static void
-initAccountBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounter)
+initAccountBinary(PGconn *con, int64_t curr, int32_t parent)
{
/*---
- * Each row of accounts table has following extra bytes:
- * - 2 bytes for number of columns
- * - 4 bytes as length for each of table's 4 columns
+ * Check documentation about COPY command:
+ * https://www.postgresql.org/docs/current/sql-copy.html
+ *
+ * Each row of accounts table is sent as:
+ * - 2 bytes for number of columns in tuple or sizeof(int16_t)
+ * - then 4 bytes or sizeof(int32_t) in front of each field with length of the field
+ *
+ * - accounts table has following columns (taking into account scale > 20000):
+ * - 8 bytes for aid column or sizeof(int64_t)
+ * - 4 bytes for bid column or sizeof(int32_t)
+ * - 4 bytes for abalance column or sizeof(int32_t)
+ * - 84 bytes for filler column (optional since no requirement for row length)
+ *---
*/
- int16_t max_row_len = 250 + 2 + 4 * 4; /* max row size is 250 for
- * int64 */
+ /* following is our max intent at the moment */
+ int16_t max_row_len = 2 + (4 + 8) + (4 + 4) + (4 + 4) + (4 + 84);
flushBuffer(con, max_row_len);
- addColumnCounter(columnCounter);
+ addColumnCounter(3);
if (scale <= SCALE_32BIT_THRESHOLD)
addInt32Column(curr + 1);
@@ -5423,6 +5457,7 @@ initAccountBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounte
addInt32Column(curr / parent + 1);
addInt32Column(0);
+ /* we don't send filler column here to minimize network traffic and WAL */
}
/*
@@ -5430,8 +5465,7 @@ initAccountBinary(PGconn *con, int64_t curr, int32_t parent, int8_t columnCounte
*/
static void
initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
- int counter, int64_t base, initRowMethodBinary init_row,
- int columnCounter)
+ int counter, int64_t base, initRowMethodBinary init_row)
{
int n;
PGresult *res;
@@ -5465,7 +5499,7 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
for (int64_t i = start; i < start + base; i++)
{
- init_row(con, i, base, columnCounter);
+ init_row(con, i, base);
if (CancelRequested)
break;
@@ -5474,6 +5508,9 @@ initPopulateTableCopyBinary(PGconn *con, char *table, char *columns,
}
res = PQgetResult(con);
+
+ Assert(bin_copy_buffer_length <= BIN_COPY_BUF_SIZE);
+
if (PQresultStatus(res) == PGRES_COPY_IN)
PQputCopyData(con, (char *) bin_copy_buffer, bin_copy_buffer_length);
else
@@ -5530,11 +5567,11 @@ initGenerateDataClientSideBinaryFrmt(PGconn *con)
executeStatement(con, "begin");
initPopulateTableCopyBinary(con, "pgbench_branches", "bid, bbalance",
- i, nbranches, initBranchBinary, 2);
+ i, nbranches, initBranchBinary);
initPopulateTableCopyBinary(con, "pgbench_tellers", "tid, bid, tbalance",
- i, ntellers, initTellerBinary, 3);
+ i, ntellers, initTellerBinary);
initPopulateTableCopyBinary(con, "pgbench_accounts", "aid, bid, abalance",
- i, naccounts, initAccountBinary, 3);
+ i, naccounts, initAccountBinary);
if (multi_xact)
executeStatement(con, "commit");
--
2.43.0
view thread (21+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected]
Subject: Re: Idea to enhance pgbench by more modes to generate data (multi-TXNs, UNNEST, COPY BINARY)
In-Reply-To: <PH0PR08MB702059923096E4BAA78FED02887FA@PH0PR08MB7020.namprd08.prod.outlook.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox