public inbox for [email protected]
help / color / mirror / Atom feed[PATCH] Remove PROC_IN_ANALYZE
7+ messages / 5 participants
[nested] [flat]
* [PATCH] Remove PROC_IN_ANALYZE
@ 2020-08-05 22:57 Alvaro Herrera <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Alvaro Herrera @ 2020-08-05 22:57 UTC (permalink / raw)
---
src/backend/commands/analyze.c | 13 +------------
src/include/storage/proc.h | 3 +--
src/include/storage/procarray.h | 7 -------
3 files changed, 2 insertions(+), 21 deletions(-)
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 924ef37c81..e0fa73ba79 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -247,11 +247,8 @@ analyze_rel(Oid relid, RangeVar *relation,
}
/*
- * OK, let's do it. First let other backends know I'm in ANALYZE.
+ * OK, let's do it. First, initialize progress reporting.
*/
- LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
- MyPgXact->vacuumFlags |= PROC_IN_ANALYZE;
- LWLockRelease(ProcArrayLock);
pgstat_progress_start_command(PROGRESS_COMMAND_ANALYZE,
RelationGetRelid(onerel));
@@ -279,14 +276,6 @@ analyze_rel(Oid relid, RangeVar *relation,
relation_close(onerel, NoLock);
pgstat_progress_end_command();
-
- /*
- * Reset my PGXACT flag. Note: we need this here, and not in vacuum_rel,
- * because the vacuum flag is cleared by the end-of-xact code.
- */
- LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
- MyPgXact->vacuumFlags &= ~PROC_IN_ANALYZE;
- LWLockRelease(ProcArrayLock);
}
/*
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index b20e2ad4f6..5ceb2494ba 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -52,7 +52,6 @@ struct XidCache
*/
#define PROC_IS_AUTOVACUUM 0x01 /* is it an autovac worker? */
#define PROC_IN_VACUUM 0x02 /* currently running lazy vacuum */
-#define PROC_IN_ANALYZE 0x04 /* currently running analyze */
#define PROC_VACUUM_FOR_WRAPAROUND 0x08 /* set by autovac only */
#define PROC_IN_LOGICAL_DECODING 0x10 /* currently doing logical
* decoding outside xact */
@@ -60,7 +59,7 @@ struct XidCache
/* flags reset at EOXact */
#define PROC_VACUUM_STATE_MASK \
- (PROC_IN_VACUUM | PROC_IN_ANALYZE | PROC_VACUUM_FOR_WRAPAROUND)
+ (PROC_IN_VACUUM | PROC_VACUUM_FOR_WRAPAROUND)
/*
* We allow a small number of "weak" relation locks (AccessShareLock,
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..01040d76e1 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -29,8 +29,6 @@
*/
#define PROCARRAY_VACUUM_FLAG 0x02 /* currently running lazy
* vacuum */
-#define PROCARRAY_ANALYZE_FLAG 0x04 /* currently running
- * analyze */
#define PROCARRAY_LOGICAL_DECODING_FLAG 0x10 /* currently doing logical
* decoding outside xact */
@@ -42,7 +40,6 @@
* have no corresponding PROC flag equivalent.
*/
#define PROCARRAY_PROC_FLAGS_MASK (PROCARRAY_VACUUM_FLAG | \
- PROCARRAY_ANALYZE_FLAG | \
PROCARRAY_LOGICAL_DECODING_FLAG)
/* Use the following flags as an input "flags" to GetOldestXmin function */
@@ -50,10 +47,6 @@
#define PROCARRAY_FLAGS_DEFAULT PROCARRAY_LOGICAL_DECODING_FLAG
/* Ignore vacuum backends */
#define PROCARRAY_FLAGS_VACUUM PROCARRAY_FLAGS_DEFAULT | PROCARRAY_VACUUM_FLAG
-/* Ignore analyze backends */
-#define PROCARRAY_FLAGS_ANALYZE PROCARRAY_FLAGS_DEFAULT | PROCARRAY_ANALYZE_FLAG
-/* Ignore both vacuum and analyze backends */
-#define PROCARRAY_FLAGS_VACUUM_ANALYZE PROCARRAY_FLAGS_DEFAULT | PROCARRAY_VACUUM_FLAG | PROCARRAY_ANALYZE_FLAG
extern Size ProcArrayShmemSize(void);
extern void CreateSharedProcArray(void);
--
2.20.1
--Kj7319i9nmIyA2yE--
^ permalink raw reply [nested|flat] 7+ messages in thread
* Crash: invalid DSA memory alloc request
@ 2024-12-12 21:28 Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Andreas 'ads' Scherbaum @ 2024-12-12 21:28 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>
Hello,
I'm running a couple of large tests, and in this particular test I have
a few million tables more.
At some point it fails, and I gathered the following trace:
2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc
request size 1073741824
2024-12-12 22:22:55.307 CET [1496210] BACKTRACE:
postgres: ads tabletest [local] CREATE TABLE(+0x15e570)
[0x6309c379c570]
postgres: ads tabletest [local] CREATE
TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4]
postgres: ads tabletest [local] CREATE
TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530]
postgres: ads tabletest [local] CREATE
TABLE(pgstat_prep_pending_entry+0x3a) [0x6309c3b0676a]
postgres: ads tabletest [local] CREATE
TABLE(pgstat_assoc_relation+0x32) [0x6309c3b086c2]
postgres: ads tabletest [local] CREATE
TABLE(StartReadBuffer+0x3c0) [0x6309c3ab9870]
postgres: ads tabletest [local] CREATE
TABLE(ReadBufferExtended+0xa1) [0x6309c3abb271]
postgres: ads tabletest [local] CREATE TABLE(+0x2c6caa)
[0x6309c3904caa]
postgres: ads tabletest [local] CREATE
TABLE(AlterSequence+0xc0) [0x6309c3905860]
postgres: ads tabletest [local] CREATE TABLE(+0x4b6336)
[0x6309c3af4336]
postgres: ads tabletest [local] CREATE
TABLE(standard_ProcessUtility+0x259) [0x6309c3af33f9]
postgres: ads tabletest [local] CREATE TABLE(+0x4b6e64)
[0x6309c3af4e64]
postgres: ads tabletest [local] CREATE
TABLE(standard_ProcessUtility+0x259) [0x6309c3af33f9]
postgres: ads tabletest [local] CREATE TABLE(+0x4b3d2f)
[0x6309c3af1d2f]
postgres: ads tabletest [local] CREATE TABLE(+0x4b3e4b)
[0x6309c3af1e4b]
postgres: ads tabletest [local] CREATE TABLE(PortalRun+0x16f)
[0x6309c3af226f]
postgres: ads tabletest [local] CREATE TABLE(+0x4b06cc)
[0x6309c3aee6cc]
postgres: ads tabletest [local] CREATE
TABLE(PostgresMain+0xf67) [0x6309c3aefa87]
postgres: ads tabletest [local] CREATE TABLE(+0x4accc5)
[0x6309c3aeacc5]
postgres: ads tabletest [local] CREATE
TABLE(postmaster_child_launch+0x8f) [0x6309c3a5b95f]
postgres: ads tabletest [local] CREATE TABLE(+0x421479)
[0x6309c3a5f479]
postgres: ads tabletest [local] CREATE
TABLE(PostmasterMain+0xd71) [0x6309c3a61251]
postgres: ads tabletest [local] CREATE TABLE(main+0x207)
[0x6309c379efc7]
/lib/x86_64-linux-gnu/libc.so.6(+0x2a1ca) [0x710c33a2a1ca]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x8b)
[0x710c33a2a28b]
postgres: ads tabletest [local] CREATE TABLE(_start+0x25)
[0x6309c379f595]
2024-12-12 22:22:55.307 CET [1496210] STATEMENT: CREATE TABLE IF NOT
EXISTS test_16718629 (id SERIAL PRIMARY KEY, d VARCHAR(200), e
VARCHAR(200), f VARCHAR(200), i INTEGER, j INTEGER);
PostgreSQL Version is 17.2, compiled with debug symbols.
tabletest=# select version();
version
--------------------------------------------------------------------------------------------------
PostgreSQL 17.2 on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu
13.2.0-23ubuntu4) 13.2.0, 64-bit
(1 row)
I'm not able to reproduce this for every DDL statement, but grouping
together about 50 of them it fails at some point.
Regards,
--
Andreas 'ads' Scherbaum
German PostgreSQL User Group
European PostgreSQL User Group - Board of Directors
Volunteer Regional Contact, Germany - PostgreSQL Project
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
@ 2024-12-12 21:49 ` Matthias van de Meent <[email protected]>
2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Matthias van de Meent @ 2024-12-12 21:49 UTC (permalink / raw)
To: Andreas 'ads' Scherbaum <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Thu, 12 Dec 2024 at 22:28, Andreas 'ads' Scherbaum <[email protected]> wrote:
>
>
> Hello,
>
> I'm running a couple of large tests, and in this particular test I have
> a few million tables more.
>
> At some point it fails, and I gathered the following trace:
>
>
> 2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc
> request size 1073741824
> 2024-12-12 22:22:55.307 CET [1496210] BACKTRACE:
> postgres: ads tabletest [local] CREATE TABLE(+0x15e570)
> [0x6309c379c570]
> postgres: ads tabletest [local] CREATE
> TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4]
> postgres: ads tabletest [local] CREATE
> TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530]
It looks like the dshash table used in the pgstats system uses
resize(), which only specifies DSA_ALLOC_ZERO, not DSA_ALLOC_HUGE,
causing issues when the table grows larger than 1 GB.
I expect that error to disappear when you replace the
dsa_allocate0(...) call in dshash.c's resize function with
dsa_allocate_extended(..., DSA_ALLOC_HUGE | DSA_ALLOC_ZERO) as
attached, but haven't tested it due to a lack of database with
millions of relations.
Kind regards,
Matthias van de Meent
Attachments:
[application/octet-stream] dshash_pgstat_fix.patch (1.1K, ../../CAEze2WhO235HKnc=qCh7GmjXBpL9LDPC6qQk4pvq57cZiDC7Pg@mail.gmail.com/2-dshash_pgstat_fix.patch)
download | inline diff:
From 5b8e5e92de0ff68e058b31f450fa55211af7c3a8 Mon Sep 17 00:00:00 2001
From: Matthias van de Meent <[email protected]>
Date: Thu, 12 Dec 2024 22:47:04 +0100
Subject: [PATCH] Fix "invalid DSA memory alloc request size" in pgstat
PGStat may use more than 1GB of DSA segments in its dshash table, so allow the DSHash system to allocate such sizes.
---
src/backend/lib/dshash.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c
index 93a9e21ddd209..c364a38f12d54 100644
--- a/src/backend/lib/dshash.c
+++ b/src/backend/lib/dshash.c
@@ -887,8 +887,9 @@ resize(dshash_table *hash_table, size_t new_size_log2)
Assert(new_size_log2 == hash_table->control->size_log2 + 1);
/* Allocate the space for the new table. */
- new_buckets_shared = dsa_allocate0(hash_table->area,
- sizeof(dsa_pointer) * new_size);
+ new_buckets_shared =
+ dsa_allocate_extended(hash_table->area, sizeof(dsa_pointer) * new_size,
+ DSA_ALLOC_HUGE | DSA_ALLOC_ZERO);
new_buckets = dsa_get_address(hash_table->area, new_buckets_shared);
/*
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
@ 2024-12-16 07:00 ` Andreas 'ads' Scherbaum <[email protected]>
2024-12-16 22:18 ` Re: Crash: invalid DSA memory alloc request Nathan Bossart <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Andreas 'ads' Scherbaum @ 2024-12-16 07:00 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
Hi Matthias,
On Thu, Dec 12, 2024 at 10:49 PM Matthias van de Meent <
[email protected]> wrote:
> On Thu, 12 Dec 2024 at 22:28, Andreas 'ads' Scherbaum <[email protected]> wrote:
> >
> >
> > Hello,
> >
> > I'm running a couple of large tests, and in this particular test I have
> > a few million tables more.
> >
> > At some point it fails, and I gathered the following trace:
> >
> >
> > 2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc
> > request size 1073741824
> > 2024-12-12 22:22:55.307 CET [1496210] BACKTRACE:
> > postgres: ads tabletest [local] CREATE TABLE(+0x15e570)
> > [0x6309c379c570]
> > postgres: ads tabletest [local] CREATE
> > TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4]
> > postgres: ads tabletest [local] CREATE
> > TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530]
>
> It looks like the dshash table used in the pgstats system uses
> resize(), which only specifies DSA_ALLOC_ZERO, not DSA_ALLOC_HUGE,
> causing issues when the table grows larger than 1 GB.
>
> I expect that error to disappear when you replace the
> dsa_allocate0(...) call in dshash.c's resize function with
> dsa_allocate_extended(..., DSA_ALLOC_HUGE | DSA_ALLOC_ZERO) as
> attached, but haven't tested it due to a lack of database with
> millions of relations.
>
Can confirm that the crash no longer happens when applying your patch.
Was able to both continue the old and crashed test, as well as run a new
test:
tabletest=# select count(*) from information_schema.tables;
count
----------
20000211
(1 row)
Thanks,
--
Andreas 'ads' Scherbaum
German PostgreSQL User Group
European PostgreSQL User Group - Board of Directors
Volunteer Regional Contact, Germany - PostgreSQL Project
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
@ 2024-12-16 22:18 ` Nathan Bossart <[email protected]>
2024-12-17 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-17 07:50 ` Re: Crash: invalid DSA memory alloc request Michael Paquier <[email protected]>
0 siblings, 2 replies; 7+ messages in thread
From: Nathan Bossart @ 2024-12-16 22:18 UTC (permalink / raw)
To: Andreas 'ads' Scherbaum <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote:
> Can confirm that the crash no longer happens when applying your patch.
The patch looks reasonable to me. I'll commit it soon unless someone
objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only
intended to catch faulty allocation requests [0].
> Was able to both continue the old and crashed test, as well as run a new
> test:
>
> tabletest=# select count(*) from information_schema.tables;
> count
> ----------
> 20000211
> (1 row)
That's a lot of tables...
[0] https://postgr.es/m/28062.1487456862%40sss.pgh.pa.us
--
nathan
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-16 22:18 ` Re: Crash: invalid DSA memory alloc request Nathan Bossart <[email protected]>
@ 2024-12-17 07:00 ` Andreas 'ads' Scherbaum <[email protected]>
1 sibling, 0 replies; 7+ messages in thread
From: Andreas 'ads' Scherbaum @ 2024-12-17 07:00 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
Hello,
On Mon, Dec 16, 2024 at 11:18 PM Nathan Bossart <[email protected]>
wrote:
> On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote:
> > Can confirm that the crash no longer happens when applying your patch.
>
> The patch looks reasonable to me. I'll commit it soon unless someone
> objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only
> intended to catch faulty allocation requests [0].
>
Is there a way to test it, except by creating so many tables?
There might be more such problems.
I did run a few basic queries in the database, but that's far from a full
test.
> > Was able to both continue the old and crashed test, as well as run a new
> > test:
> >
> > tabletest=# select count(*) from information_schema.tables;
> > count
> > ----------
> > 20000211
> > (1 row)
>
> That's a lot of tables...
>
Started as a discussion, got me curious and it's only about a magnitude or
so off
from what I've seen in production.
Not unrealistic to find out when and where it breaks.
Thanks,
--
Andreas 'ads' Scherbaum
German PostgreSQL User Group
European PostgreSQL User Group - Board of Directors
Volunteer Regional Contact, Germany - PostgreSQL Project
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-16 22:18 ` Re: Crash: invalid DSA memory alloc request Nathan Bossart <[email protected]>
@ 2024-12-17 07:50 ` Michael Paquier <[email protected]>
1 sibling, 0 replies; 7+ messages in thread
From: Michael Paquier @ 2024-12-17 07:50 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Andreas 'ads' Scherbaum <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Dec 16, 2024 at 04:18:26PM -0600, Nathan Bossart wrote:
> On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote:
>> Can confirm that the crash no longer happens when applying your patch.
>
> The patch looks reasonable to me. I'll commit it soon unless someone
> objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only
> intended to catch faulty allocation requests [0].
No objections.
Most likely this issue gets by a large degree easier to reach now that
we can plug into the backend custom pgstats kinds. If pgstats or an
equivalent implementation uses pgstats, I don't think that we'll be
able to live without lifting this limit (500k query entries are
common, at 2kB each it would be enough to blow things), so using
DSA_ALLOC_HUGE sounds good to me. I don't see a huge point in
backpatching, FWIW.
>> 20000211
>
> That's a lot of tables...
>
> [0] https://postgr.es/m/28062.1487456862%40sss.pgh.pa.us
And most likely don't do that. If you want to play more in this area,
there is also the join-1M-tables-in-a-single-query game.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2024-12-17 07:50 UTC | newest]
Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-08-05 22:57 [PATCH] Remove PROC_IN_ANALYZE Alvaro Herrera <[email protected]>
2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]>
2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-16 22:18 ` Re: Crash: invalid DSA memory alloc request Nathan Bossart <[email protected]>
2024-12-17 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]>
2024-12-17 07:50 ` Re: Crash: invalid DSA memory alloc request Michael Paquier <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox