public inbox for [email protected]help / color / mirror / Atom feed
[PATCH] Remove PROC_IN_ANALYZE 7+ messages / 5 participants [nested] [flat]
* [PATCH] Remove PROC_IN_ANALYZE @ 2020-08-05 22:57 Alvaro Herrera <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Alvaro Herrera @ 2020-08-05 22:57 UTC (permalink / raw) --- src/backend/commands/analyze.c | 13 +------------ src/include/storage/proc.h | 3 +-- src/include/storage/procarray.h | 7 ------- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 924ef37c81..e0fa73ba79 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -247,11 +247,8 @@ analyze_rel(Oid relid, RangeVar *relation, } /* - * OK, let's do it. First let other backends know I'm in ANALYZE. + * OK, let's do it. First, initialize progress reporting. */ - LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); - MyPgXact->vacuumFlags |= PROC_IN_ANALYZE; - LWLockRelease(ProcArrayLock); pgstat_progress_start_command(PROGRESS_COMMAND_ANALYZE, RelationGetRelid(onerel)); @@ -279,14 +276,6 @@ analyze_rel(Oid relid, RangeVar *relation, relation_close(onerel, NoLock); pgstat_progress_end_command(); - - /* - * Reset my PGXACT flag. Note: we need this here, and not in vacuum_rel, - * because the vacuum flag is cleared by the end-of-xact code. - */ - LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); - MyPgXact->vacuumFlags &= ~PROC_IN_ANALYZE; - LWLockRelease(ProcArrayLock); } /* diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index b20e2ad4f6..5ceb2494ba 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -52,7 +52,6 @@ struct XidCache */ #define PROC_IS_AUTOVACUUM 0x01 /* is it an autovac worker? */ #define PROC_IN_VACUUM 0x02 /* currently running lazy vacuum */ -#define PROC_IN_ANALYZE 0x04 /* currently running analyze */ #define PROC_VACUUM_FOR_WRAPAROUND 0x08 /* set by autovac only */ #define PROC_IN_LOGICAL_DECODING 0x10 /* currently doing logical * decoding outside xact */ @@ -60,7 +59,7 @@ struct XidCache /* flags reset at EOXact */ #define PROC_VACUUM_STATE_MASK \ - (PROC_IN_VACUUM | PROC_IN_ANALYZE | PROC_VACUUM_FOR_WRAPAROUND) + (PROC_IN_VACUUM | PROC_VACUUM_FOR_WRAPAROUND) /* * We allow a small number of "weak" relation locks (AccessShareLock, diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index a5c7d0c064..01040d76e1 100644 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -29,8 +29,6 @@ */ #define PROCARRAY_VACUUM_FLAG 0x02 /* currently running lazy * vacuum */ -#define PROCARRAY_ANALYZE_FLAG 0x04 /* currently running - * analyze */ #define PROCARRAY_LOGICAL_DECODING_FLAG 0x10 /* currently doing logical * decoding outside xact */ @@ -42,7 +40,6 @@ * have no corresponding PROC flag equivalent. */ #define PROCARRAY_PROC_FLAGS_MASK (PROCARRAY_VACUUM_FLAG | \ - PROCARRAY_ANALYZE_FLAG | \ PROCARRAY_LOGICAL_DECODING_FLAG) /* Use the following flags as an input "flags" to GetOldestXmin function */ @@ -50,10 +47,6 @@ #define PROCARRAY_FLAGS_DEFAULT PROCARRAY_LOGICAL_DECODING_FLAG /* Ignore vacuum backends */ #define PROCARRAY_FLAGS_VACUUM PROCARRAY_FLAGS_DEFAULT | PROCARRAY_VACUUM_FLAG -/* Ignore analyze backends */ -#define PROCARRAY_FLAGS_ANALYZE PROCARRAY_FLAGS_DEFAULT | PROCARRAY_ANALYZE_FLAG -/* Ignore both vacuum and analyze backends */ -#define PROCARRAY_FLAGS_VACUUM_ANALYZE PROCARRAY_FLAGS_DEFAULT | PROCARRAY_VACUUM_FLAG | PROCARRAY_ANALYZE_FLAG extern Size ProcArrayShmemSize(void); extern void CreateSharedProcArray(void); -- 2.20.1 --Kj7319i9nmIyA2yE-- ^ permalink raw reply [nested|flat] 7+ messages in thread
* Crash: invalid DSA memory alloc request @ 2024-12-12 21:28 Andreas 'ads' Scherbaum <[email protected]> 0 siblings, 1 reply; 7+ messages in thread From: Andreas 'ads' Scherbaum @ 2024-12-12 21:28 UTC (permalink / raw) To: PostgreSQL Hackers <[email protected]> Hello, I'm running a couple of large tests, and in this particular test I have a few million tables more. At some point it fails, and I gathered the following trace: 2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc request size 1073741824 2024-12-12 22:22:55.307 CET [1496210] BACKTRACE: postgres: ads tabletest [local] CREATE TABLE(+0x15e570) [0x6309c379c570] postgres: ads tabletest [local] CREATE TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4] postgres: ads tabletest [local] CREATE TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530] postgres: ads tabletest [local] CREATE TABLE(pgstat_prep_pending_entry+0x3a) [0x6309c3b0676a] postgres: ads tabletest [local] CREATE TABLE(pgstat_assoc_relation+0x32) [0x6309c3b086c2] postgres: ads tabletest [local] CREATE TABLE(StartReadBuffer+0x3c0) [0x6309c3ab9870] postgres: ads tabletest [local] CREATE TABLE(ReadBufferExtended+0xa1) [0x6309c3abb271] postgres: ads tabletest [local] CREATE TABLE(+0x2c6caa) [0x6309c3904caa] postgres: ads tabletest [local] CREATE TABLE(AlterSequence+0xc0) [0x6309c3905860] postgres: ads tabletest [local] CREATE TABLE(+0x4b6336) [0x6309c3af4336] postgres: ads tabletest [local] CREATE TABLE(standard_ProcessUtility+0x259) [0x6309c3af33f9] postgres: ads tabletest [local] CREATE TABLE(+0x4b6e64) [0x6309c3af4e64] postgres: ads tabletest [local] CREATE TABLE(standard_ProcessUtility+0x259) [0x6309c3af33f9] postgres: ads tabletest [local] CREATE TABLE(+0x4b3d2f) [0x6309c3af1d2f] postgres: ads tabletest [local] CREATE TABLE(+0x4b3e4b) [0x6309c3af1e4b] postgres: ads tabletest [local] CREATE TABLE(PortalRun+0x16f) [0x6309c3af226f] postgres: ads tabletest [local] CREATE TABLE(+0x4b06cc) [0x6309c3aee6cc] postgres: ads tabletest [local] CREATE TABLE(PostgresMain+0xf67) [0x6309c3aefa87] postgres: ads tabletest [local] CREATE TABLE(+0x4accc5) [0x6309c3aeacc5] postgres: ads tabletest [local] CREATE TABLE(postmaster_child_launch+0x8f) [0x6309c3a5b95f] postgres: ads tabletest [local] CREATE TABLE(+0x421479) [0x6309c3a5f479] postgres: ads tabletest [local] CREATE TABLE(PostmasterMain+0xd71) [0x6309c3a61251] postgres: ads tabletest [local] CREATE TABLE(main+0x207) [0x6309c379efc7] /lib/x86_64-linux-gnu/libc.so.6(+0x2a1ca) [0x710c33a2a1ca] /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x8b) [0x710c33a2a28b] postgres: ads tabletest [local] CREATE TABLE(_start+0x25) [0x6309c379f595] 2024-12-12 22:22:55.307 CET [1496210] STATEMENT: CREATE TABLE IF NOT EXISTS test_16718629 (id SERIAL PRIMARY KEY, d VARCHAR(200), e VARCHAR(200), f VARCHAR(200), i INTEGER, j INTEGER); PostgreSQL Version is 17.2, compiled with debug symbols. tabletest=# select version(); version -------------------------------------------------------------------------------------------------- PostgreSQL 17.2 on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 13.2.0-23ubuntu4) 13.2.0, 64-bit (1 row) I'm not able to reproduce this for every DDL statement, but grouping together about 50 of them it fails at some point. Regards, -- Andreas 'ads' Scherbaum German PostgreSQL User Group European PostgreSQL User Group - Board of Directors Volunteer Regional Contact, Germany - PostgreSQL Project ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request @ 2024-12-12 21:49 Matthias van de Meent <[email protected]> parent: Andreas 'ads' Scherbaum <[email protected]> 0 siblings, 1 reply; 7+ messages in thread From: Matthias van de Meent @ 2024-12-12 21:49 UTC (permalink / raw) To: Andreas 'ads' Scherbaum <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> On Thu, 12 Dec 2024 at 22:28, Andreas 'ads' Scherbaum <[email protected]> wrote: > > > Hello, > > I'm running a couple of large tests, and in this particular test I have > a few million tables more. > > At some point it fails, and I gathered the following trace: > > > 2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc > request size 1073741824 > 2024-12-12 22:22:55.307 CET [1496210] BACKTRACE: > postgres: ads tabletest [local] CREATE TABLE(+0x15e570) > [0x6309c379c570] > postgres: ads tabletest [local] CREATE > TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4] > postgres: ads tabletest [local] CREATE > TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530] It looks like the dshash table used in the pgstats system uses resize(), which only specifies DSA_ALLOC_ZERO, not DSA_ALLOC_HUGE, causing issues when the table grows larger than 1 GB. I expect that error to disappear when you replace the dsa_allocate0(...) call in dshash.c's resize function with dsa_allocate_extended(..., DSA_ALLOC_HUGE | DSA_ALLOC_ZERO) as attached, but haven't tested it due to a lack of database with millions of relations. Kind regards, Matthias van de Meent Attachments: [application/octet-stream] dshash_pgstat_fix.patch (1.1K, ../../CAEze2WhO235HKnc=qCh7GmjXBpL9LDPC6qQk4pvq57cZiDC7Pg@mail.gmail.com/2-dshash_pgstat_fix.patch) download | inline diff: From 5b8e5e92de0ff68e058b31f450fa55211af7c3a8 Mon Sep 17 00:00:00 2001 From: Matthias van de Meent <[email protected]> Date: Thu, 12 Dec 2024 22:47:04 +0100 Subject: [PATCH] Fix "invalid DSA memory alloc request size" in pgstat PGStat may use more than 1GB of DSA segments in its dshash table, so allow the DSHash system to allocate such sizes. --- src/backend/lib/dshash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c index 93a9e21ddd209..c364a38f12d54 100644 --- a/src/backend/lib/dshash.c +++ b/src/backend/lib/dshash.c @@ -887,8 +887,9 @@ resize(dshash_table *hash_table, size_t new_size_log2) Assert(new_size_log2 == hash_table->control->size_log2 + 1); /* Allocate the space for the new table. */ - new_buckets_shared = dsa_allocate0(hash_table->area, - sizeof(dsa_pointer) * new_size); + new_buckets_shared = + dsa_allocate_extended(hash_table->area, sizeof(dsa_pointer) * new_size, + DSA_ALLOC_HUGE | DSA_ALLOC_ZERO); new_buckets = dsa_get_address(hash_table->area, new_buckets_shared); /* ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request @ 2024-12-16 07:00 Andreas 'ads' Scherbaum <[email protected]> parent: Matthias van de Meent <[email protected]> 0 siblings, 1 reply; 7+ messages in thread From: Andreas 'ads' Scherbaum @ 2024-12-16 07:00 UTC (permalink / raw) To: Matthias van de Meent <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> Hi Matthias, On Thu, Dec 12, 2024 at 10:49 PM Matthias van de Meent < [email protected]> wrote: > On Thu, 12 Dec 2024 at 22:28, Andreas 'ads' Scherbaum <[email protected]> wrote: > > > > > > Hello, > > > > I'm running a couple of large tests, and in this particular test I have > > a few million tables more. > > > > At some point it fails, and I gathered the following trace: > > > > > > 2024-12-12 22:22:55.307 CET [1496210] ERROR: invalid DSA memory alloc > > request size 1073741824 > > 2024-12-12 22:22:55.307 CET [1496210] BACKTRACE: > > postgres: ads tabletest [local] CREATE TABLE(+0x15e570) > > [0x6309c379c570] > > postgres: ads tabletest [local] CREATE > > TABLE(dshash_find_or_insert+0x1a4) [0x6309c39882d4] > > postgres: ads tabletest [local] CREATE > > TABLE(pgstat_get_entry_ref+0x440) [0x6309c3b0a530] > > It looks like the dshash table used in the pgstats system uses > resize(), which only specifies DSA_ALLOC_ZERO, not DSA_ALLOC_HUGE, > causing issues when the table grows larger than 1 GB. > > I expect that error to disappear when you replace the > dsa_allocate0(...) call in dshash.c's resize function with > dsa_allocate_extended(..., DSA_ALLOC_HUGE | DSA_ALLOC_ZERO) as > attached, but haven't tested it due to a lack of database with > millions of relations. > Can confirm that the crash no longer happens when applying your patch. Was able to both continue the old and crashed test, as well as run a new test: tabletest=# select count(*) from information_schema.tables; count ---------- 20000211 (1 row) Thanks, -- Andreas 'ads' Scherbaum German PostgreSQL User Group European PostgreSQL User Group - Board of Directors Volunteer Regional Contact, Germany - PostgreSQL Project ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request @ 2024-12-16 22:18 Nathan Bossart <[email protected]> parent: Andreas 'ads' Scherbaum <[email protected]> 0 siblings, 2 replies; 7+ messages in thread From: Nathan Bossart @ 2024-12-16 22:18 UTC (permalink / raw) To: Andreas 'ads' Scherbaum <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote: > Can confirm that the crash no longer happens when applying your patch. The patch looks reasonable to me. I'll commit it soon unless someone objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only intended to catch faulty allocation requests [0]. > Was able to both continue the old and crashed test, as well as run a new > test: > > tabletest=# select count(*) from information_schema.tables; > count > ---------- > 20000211 > (1 row) That's a lot of tables... [0] https://postgr.es/m/28062.1487456862%40sss.pgh.pa.us -- nathan ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request @ 2024-12-17 07:00 Andreas 'ads' Scherbaum <[email protected]> parent: Nathan Bossart <[email protected]> 1 sibling, 0 replies; 7+ messages in thread From: Andreas 'ads' Scherbaum @ 2024-12-17 07:00 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]> Hello, On Mon, Dec 16, 2024 at 11:18 PM Nathan Bossart <[email protected]> wrote: > On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote: > > Can confirm that the crash no longer happens when applying your patch. > > The patch looks reasonable to me. I'll commit it soon unless someone > objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only > intended to catch faulty allocation requests [0]. > Is there a way to test it, except by creating so many tables? There might be more such problems. I did run a few basic queries in the database, but that's far from a full test. > > Was able to both continue the old and crashed test, as well as run a new > > test: > > > > tabletest=# select count(*) from information_schema.tables; > > count > > ---------- > > 20000211 > > (1 row) > > That's a lot of tables... > Started as a discussion, got me curious and it's only about a magnitude or so off from what I've seen in production. Not unrealistic to find out when and where it breaks. Thanks, -- Andreas 'ads' Scherbaum German PostgreSQL User Group European PostgreSQL User Group - Board of Directors Volunteer Regional Contact, Germany - PostgreSQL Project ^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Crash: invalid DSA memory alloc request @ 2024-12-17 07:50 Michael Paquier <[email protected]> parent: Nathan Bossart <[email protected]> 1 sibling, 0 replies; 7+ messages in thread From: Michael Paquier @ 2024-12-17 07:50 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: Andreas 'ads' Scherbaum <[email protected]>; Matthias van de Meent <[email protected]>; PostgreSQL Hackers <[email protected]> On Mon, Dec 16, 2024 at 04:18:26PM -0600, Nathan Bossart wrote: > On Mon, Dec 16, 2024 at 08:00:00AM +0100, Andreas 'ads' Scherbaum wrote: >> Can confirm that the crash no longer happens when applying your patch. > > The patch looks reasonable to me. I'll commit it soon unless someone > objects. I was surprised to learn that the DSA_ALLOC_HUGE flag is only > intended to catch faulty allocation requests [0]. No objections. Most likely this issue gets by a large degree easier to reach now that we can plug into the backend custom pgstats kinds. If pgstats or an equivalent implementation uses pgstats, I don't think that we'll be able to live without lifting this limit (500k query entries are common, at 2kB each it would be enough to blow things), so using DSA_ALLOC_HUGE sounds good to me. I don't see a huge point in backpatching, FWIW. >> 20000211 > > That's a lot of tables... > > [0] https://postgr.es/m/28062.1487456862%40sss.pgh.pa.us And most likely don't do that. If you want to play more in this area, there is also the join-1M-tables-in-a-single-query game. -- Michael Attachments: [application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc) download ^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2024-12-17 07:50 UTC | newest] Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-08-05 22:57 [PATCH] Remove PROC_IN_ANALYZE Alvaro Herrera <[email protected]> 2024-12-12 21:28 Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]> 2024-12-12 21:49 ` Re: Crash: invalid DSA memory alloc request Matthias van de Meent <[email protected]> 2024-12-16 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]> 2024-12-16 22:18 ` Re: Crash: invalid DSA memory alloc request Nathan Bossart <[email protected]> 2024-12-17 07:00 ` Re: Crash: invalid DSA memory alloc request Andreas 'ads' Scherbaum <[email protected]> 2024-12-17 07:50 ` Re: Crash: invalid DSA memory alloc request Michael Paquier <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox