public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 1/4] bootstrap: convert Typ to a List* 4+ messages / 3 participants [nested] [flat]
* [PATCH 1/4] bootstrap: convert Typ to a List* @ 2020-11-20 02:48 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Justin Pryzby @ 2020-11-20 02:48 UTC (permalink / raw) --- src/backend/bootstrap/bootstrap.c | 89 ++++++++++++++----------------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 6f615e6622..1b940d9d27 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -58,7 +58,7 @@ static void BootstrapModeMain(void); static void bootstrap_signals(void); static void ShutdownAuxiliaryProcess(int code, Datum arg); static Form_pg_attribute AllocateAttribute(void); -static void populate_typ_array(void); +static void populate_typ(void); static Oid gettype(char *type); static void cleanup(void); @@ -159,7 +159,7 @@ struct typmap FormData_pg_type am_typ; }; -static struct typmap **Typ = NULL; +static List *Typ = NIL; /* List of struct typmap* */ static struct typmap *Ap = NULL; static Datum values[MAXATTR]; /* current row's attribute values */ @@ -595,10 +595,10 @@ boot_openrel(char *relname) /* * pg_type must be filled before any OPEN command is executed, hence we - * can now populate the Typ array if we haven't yet. + * can now populate Typ if we haven't yet. */ - if (Typ == NULL) - populate_typ_array(); + if (Typ == NIL) + populate_typ(); if (boot_reldesc != NULL) closerel(NULL); @@ -688,7 +688,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness) typeoid = gettype(type); - if (Typ != NULL) + if (Typ != NIL) { attrtypes[attnum]->atttypid = Ap->am_oid; attrtypes[attnum]->attlen = Ap->am_typ.typlen; @@ -866,47 +866,36 @@ cleanup(void) } /* ---------------- - * populate_typ_array + * populate_typ * - * Load the Typ array by reading pg_type. + * Load Typ by reading pg_type. * ---------------- */ static void -populate_typ_array(void) +populate_typ(void) { Relation rel; TableScanDesc scan; HeapTuple tup; - int nalloc; - int i; - - Assert(Typ == NULL); + MemoryContext old; - nalloc = 512; - Typ = (struct typmap **) - MemoryContextAlloc(TopMemoryContext, nalloc * sizeof(struct typmap *)); + Assert(Typ == NIL); rel = table_open(TypeRelationId, NoLock); scan = table_beginscan_catalog(rel, 0, NULL); - i = 0; + old = MemoryContextSwitchTo(TopMemoryContext); while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_type typForm = (Form_pg_type) GETSTRUCT(tup); + struct typmap *newtyp; - /* make sure there will be room for a trailing NULL pointer */ - if (i >= nalloc - 1) - { - nalloc *= 2; - Typ = (struct typmap **) - repalloc(Typ, nalloc * sizeof(struct typmap *)); - } - Typ[i] = (struct typmap *) - MemoryContextAlloc(TopMemoryContext, sizeof(struct typmap)); - Typ[i]->am_oid = typForm->oid; - memcpy(&(Typ[i]->am_typ), typForm, sizeof(Typ[i]->am_typ)); - i++; + newtyp = (struct typmap *) palloc(sizeof(struct typmap)); + Typ = lappend(Typ, newtyp); + + newtyp->am_oid = typForm->oid; + memcpy(&newtyp->am_typ, typForm, sizeof(newtyp->am_typ)); } - Typ[i] = NULL; /* Fill trailing NULL pointer */ + MemoryContextSwitchTo(old); table_endscan(scan); table_close(rel, NoLock); } @@ -916,25 +905,26 @@ populate_typ_array(void) * * NB: this is really ugly; it will return an integer index into TypInfo[], * and not an OID at all, until the first reference to a type not known in - * TypInfo[]. At that point it will read and cache pg_type in the Typ array, + * TypInfo[]. At that point it will read and cache pg_type in Typ, * and subsequently return a real OID (and set the global pointer Ap to * point at the found row in Typ). So caller must check whether Typ is - * still NULL to determine what the return value is! + * still NIL to determine what the return value is! * ---------------- */ static Oid gettype(char *type) { - if (Typ != NULL) + if (Typ != NIL) { - struct typmap **app; + ListCell *lc; - for (app = Typ; *app != NULL; app++) + foreach (lc, Typ) { - if (strncmp(NameStr((*app)->am_typ.typname), type, NAMEDATALEN) == 0) + struct typmap *app = lfirst(lc); + if (strncmp(NameStr(app->am_typ.typname), type, NAMEDATALEN) == 0) { - Ap = *app; - return (*app)->am_oid; + Ap = app; + return app->am_oid; } } } @@ -949,7 +939,7 @@ gettype(char *type) } /* Not in TypInfo, so we'd better be able to read pg_type now */ elog(DEBUG4, "external type: %s", type); - populate_typ_array(); + populate_typ(); return gettype(type); } elog(ERROR, "unrecognized type \"%s\"", type); @@ -977,17 +967,20 @@ boot_get_type_io_data(Oid typid, Oid *typinput, Oid *typoutput) { - if (Typ != NULL) + if (Typ != NIL) { /* We have the boot-time contents of pg_type, so use it */ - struct typmap **app; - struct typmap *ap; - - app = Typ; - while (*app && (*app)->am_oid != typid) - ++app; - ap = *app; - if (ap == NULL) + struct typmap *ap = NULL; + ListCell *lc; + + foreach (lc, Typ) + { + ap = lfirst(lc); + if (ap->am_oid == typid) + break; + } + + if (!ap || ap->am_oid != typid) elog(ERROR, "type OID %u not found in Typ list", typid); *typlen = ap->am_typ.typlen; -- 2.26.2 --------------BCA1373AE29B32B57608F4EE Content-Type: text/x-patch; charset=UTF-8; name="0002-Allow-composite-types-in-bootstrap-20210307.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="0002-Allow-composite-types-in-bootstrap-20210307.patch" ^ permalink raw reply [nested|flat] 4+ messages in thread
* pg16: invalid page/page verification failed @ 2023-10-05 16:45 Justin Pryzby <[email protected]> 0 siblings, 1 reply; 4+ messages in thread From: Justin Pryzby @ 2023-10-05 16:45 UTC (permalink / raw) To: [email protected] On an instance running pg16.0: log_time | 2023-10-05 10:03:00.014-05 backend_type | autovacuum worker left | page verification failed, calculated checksum 5074 but expected 5050 context | while scanning block 119 of relation "public.postgres_log_2023_10_05_0900" This is the only error I've seen so far, and for all I know there's a issue on the storage behind the VM, or a cosmic ray hit. But I moved the table out of the way and saved a copy of get_raw_page() in case someone wants to ask about it. public | BROKEN_postgres_log_2023_10_05_0900 | table | postgres | permanent | heap | 1664 kB This table is what it sounds like: a partition into which CSV logs are COPY'ed. It would've been created around 8am. There's no special params set for the table nor for autovacuum. Although we have a ZFS tablespace, these tables aren't on it, and full_page_writes=on. There's no crashes, and the instance has been up since it was pg_upgraded from v15.4 on sep25. pg_stat_all_tables indicates that the table was never (successfully) vacuumed. This was compiled to RPM on centos7, and might include a few commits made since v16.0. postgres=# SELECT * FROM heap_page_item_attrs(get_raw_page(801594131::regclass::text, 119), 801594131); lp | lp_off | lp_flags | lp_len | t_xmin | t_xmax | t_field3 | t_ctid | t_infomask2 | t_infomask | t_hoff | t_bits | t_oid | t_attrs 1 | 2304 | 1 | 16 | | | | | | | | | | 2 | 8160 | 1 | 16 | | | | | | | | | | 3 | 8144 | 1 | 16 | | | | | | | | | | ...all the same except for lp_off... 365 | 2352 | 1 | 16 | | | | | | | | | | 366 | 2336 | 1 | 16 | | | | | | | | | | 367 | 2320 | 1 | 16 | | | | | | | | | | postgres=# SELECT FROM (SELECT tuple_data_split(801594131, t_data, t_infomask, t_infomask2, t_bits) a FROM heap_page_items(get_raw_page(801594131::regclass::text, 119))) WHERE a IS NOT NULL; WARNING: page verification failed, calculated checksum 5074 but expected 5050 (0 rows) -- Justin ^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: pg16: invalid page/page verification failed @ 2023-10-05 17:16 Matthias van de Meent <[email protected]> parent: Justin Pryzby <[email protected]> 0 siblings, 1 reply; 4+ messages in thread From: Matthias van de Meent @ 2023-10-05 17:16 UTC (permalink / raw) To: Justin Pryzby <[email protected]>; +Cc: [email protected] On Thu, 5 Oct 2023 at 18:48, Justin Pryzby <[email protected]> wrote: > > On an instance running pg16.0: > > log_time | 2023-10-05 10:03:00.014-05 > backend_type | autovacuum worker > left | page verification failed, calculated checksum 5074 but expected 5050 > context | while scanning block 119 of relation "public.postgres_log_2023_10_05_0900" > > This is the only error I've seen so far, and for all I know there's a > issue on the storage behind the VM, or a cosmic ray hit. But I moved > the table out of the way and saved a copy of get_raw_page() in case > someone wants to ask about it. > > postgres=# SELECT * FROM heap_page_item_attrs(get_raw_page(801594131::regclass::text, 119), 801594131); > lp | lp_off | lp_flags | lp_len | t_xmin | t_xmax | t_field3 | t_ctid | t_infomask2 | t_infomask | t_hoff | t_bits | t_oid | t_attrs > 1 | 2304 | 1 | 16 | | | | | | | | | | > 2 | 8160 | 1 | 16 | | | | | | | | | | > 3 | 8144 | 1 | 16 | | | | | | | | | | > ...all the same except for lp_off... > 365 | 2352 | 1 | 16 | | | | | | | | | | > 366 | 2336 | 1 | 16 | | | | | | | | | | > 367 | 2320 | 1 | 16 | | | | | | | | | | That's not a HEAP page; it looks more like a btree page: lp_len is too short for heap (which starts at lp_len = 24), and there are too many line pointers for an 8KiB heap page. btree often has lp_len of 16: 8 bytes indextuple header, one maxalign of data (e.g. int or bigint). So, assuming it's a block of a different relation kind, then it's also likely it was originally located elsewhere in that other relation, indeed causing the checksum failure. You can further validate this by looking at the page header's pd_special value - if it is 8176, that'd be another indicator for it being a btree. Kind regards, Matthias van de Meent. ^ permalink raw reply [nested|flat] 4+ messages in thread
* Re: pg16: invalid page/page verification failed @ 2023-10-05 18:48 Justin Pryzby <[email protected]> parent: Matthias van de Meent <[email protected]> 0 siblings, 0 replies; 4+ messages in thread From: Justin Pryzby @ 2023-10-05 18:48 UTC (permalink / raw) To: Matthias van de Meent <[email protected]>; +Cc: [email protected] On Thu, Oct 05, 2023 at 07:16:31PM +0200, Matthias van de Meent wrote: > On Thu, 5 Oct 2023 at 18:48, Justin Pryzby <[email protected]> wrote: > > > > On an instance running pg16.0: > > > > log_time | 2023-10-05 10:03:00.014-05 > > backend_type | autovacuum worker > > left | page verification failed, calculated checksum 5074 but expected 5050 > > context | while scanning block 119 of relation "public.postgres_log_2023_10_05_0900" > > > > This is the only error I've seen so far, and for all I know there's a > > issue on the storage behind the VM, or a cosmic ray hit. But I moved > > the table out of the way and saved a copy of get_raw_page() in case > > someone wants to ask about it. > > > > postgres=# SELECT * FROM heap_page_item_attrs(get_raw_page(801594131::regclass::text, 119), 801594131); > > lp | lp_off | lp_flags | lp_len | t_xmin | t_xmax | t_field3 | t_ctid | t_infomask2 | t_infomask | t_hoff | t_bits | t_oid | t_attrs > > 1 | 2304 | 1 | 16 | | | | | | | | | | > > 2 | 8160 | 1 | 16 | | | | | | | | | | > > 3 | 8144 | 1 | 16 | | | | | | | | | | > > ...all the same except for lp_off... > > 365 | 2352 | 1 | 16 | | | | | | | | | | > > 366 | 2336 | 1 | 16 | | | | | | | | | | > > 367 | 2320 | 1 | 16 | | | | | | | | | | > > That's not a HEAP page; it looks more like a btree page: lp_len is too > short for heap (which starts at lp_len = 24), and there are too many > line pointers for an 8KiB heap page. btree often has lp_len of 16: 8 > bytes indextuple header, one maxalign of data (e.g. int or bigint). > > So, assuming it's a block of a different relation kind, then it's also > likely it was originally located elsewhere in that other relation, > indeed causing the checksum failure. You can further validate this by > looking at the page header's pd_special value - if it is 8176, that'd > be another indicator for it being a btree. Nice point. postgres=# SET ignore_checksum_failure=on; SELECT * FROM generate_series(115,119) AS a, page_header(get_raw_page(801594131::regclass::text, a)) AS b; WARNING: page verification failed, calculated checksum 5074 but expected 5050 a | lsn | checksum | flags | lower | upper | special | pagesize | version | prune_xid -----+--------------+----------+-------+-------+-------+---------+----------+---------+----------- 115 | B61/A9436C8 | -23759 | 4 | 92 | 336 | 8192 | 8192 | 4 | 0 116 | B61/A944FA0 | 3907 | 4 | 104 | 224 | 8192 | 8192 | 4 | 0 117 | B61/A946828 | -24448 | 4 | 76 | 264 | 8192 | 8192 | 4 | 0 118 | B61/A94CCE0 | 26915 | 4 | 28 | 6256 | 8192 | 8192 | 4 | 0 119 | B5C/9F30D1C8 | 5050 | 0 | 1492 | 2304 | 8176 | 8192 | 4 | 0 The table itself has a few btree indexes on text columns and a brin index on log_timestamp, but not on the integers. It sounds like it's what's expected at this point, but after I "SET ignore_checksum_failure=on", and read the page in, vacuum kicked off and then crashed (in heap_page_prune() if that half of the stack trace can be trusted). *** stack smashing detected ***: postgres: autovacuum worker postgres terminated < 2023-10-05 12:35:30.764 CDT >LOG: server process (PID 30692) was terminated by signal 11: Segmentation fault < 2023-10-05 12:35:30.764 CDT >DETAIL: Failed process was running: autovacuum: VACUUM ANALYZE public.BROKEN_postgres_log_2023_10_05_0900 I took the opportunity to fsck the FS, which showed no errors. I was curious if the relfilenodes had gotten confused/corrupted/?? But this seems to indicate not; the problem is only one block. postgres=# SELECT oid, relfilenode, oid=relfilenode, relname FROM pg_class WHERE oid BETWEEN 801550000 AND 801594199 ORDER BY 1; oid | relfilenode | ?column? | relname -----------+-------------+----------+------------------------------------------------- 801564542 | 801564542 | t | postgres_log_2023_10_05_0800 801564545 | 801564545 | t | pg_toast_801564542 801564546 | 801564546 | t | pg_toast_801564542_index 801564547 | 801564547 | t | postgres_log_2023_10_05_0800_log_time_idx 801564548 | 801564548 | t | postgres_log_2023_10_05_0800_error_severity_idx 801564549 | 801564549 | t | postgres_log_2023_10_05_0800_error_message_idx 801564550 | 801564550 | t | postgres_log_2023_10_05_0800_duration_idx 801564551 | 801564551 | t | postgres_log_2023_10_05_0800_tempfile_idx 801594131 | 801594131 | t | BROKEN_postgres_log_2023_10_05_0900 801594134 | 801594134 | t | pg_toast_801594131 801594135 | 801594135 | t | pg_toast_801594131_index 801594136 | 801594136 | t | postgres_log_2023_10_05_0900_log_time_idx 801594137 | 801594137 | t | postgres_log_2023_10_05_0900_error_severity_idx 801594138 | 801594138 | t | postgres_log_2023_10_05_0900_error_message_idx 801594139 | 801594139 | t | postgres_log_2023_10_05_0900_duration_idx 801594140 | 801594140 | t | postgres_log_2023_10_05_0900_tempfile_idx Before anybody asks, we didn't retain WAL from this morning. FYI, the storage is ext4/LVM/scsi (it looks like this didn't use vmw_pvscsi but an emulated hardware driver). /dev/mapper/data-postgres on /var/lib/pgsql type ext4 (rw,relatime,seclabel,data=ordered) [ 0.000000] Linux version 3.10.0-1160.el7.x86_64 ([email protected]) (gcc version 4.8.5 20150623 (Red Hat 4.8.5-44) (GCC) ) #1 SMP Mon Oct 19 16:18:59 UTC 2020 [ 1.446380] scsi 2:0:1:0: Direct-Access VMware Virtual disk 1.0 PQ: 0 ANSI: 2 [ 1.470764] scsi target2:0:1: Beginning Domain Validation [ 1.471077] scsi target2:0:1: Domain Validation skipping write tests [ 1.471079] scsi target2:0:1: Ending Domain Validation [ 1.471099] scsi target2:0:1: FAST-40 WIDE SCSI 80.0 MB/s ST (25 ns, offset 127) [ 1.484109] sd 2:0:1:0: [sdb] 1048576000 512-byte logical blocks: (536 GB/500 GiB) [ 1.484136] sd 2:0:1:0: [sdb] Write Protect is off [ 1.484139] sd 2:0:1:0: [sdb] Mode Sense: 45 00 00 00 [ 1.484163] sd 2:0:1:0: [sdb] Write cache: disabled, read cache: disabled, doesn't support DPO or FUA [ 1.485808] sd 2:0:1:0: [sdb] Attached SCSI disk [ 4.271339] sd 2:0:1:0: Attached scsi generic sg1 type 0 -- Justin ^ permalink raw reply [nested|flat] 4+ messages in thread
end of thread, other threads:[~2023-10-05 18:48 UTC | newest] Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-11-20 02:48 [PATCH 1/4] bootstrap: convert Typ to a List* Justin Pryzby <[email protected]> 2023-10-05 16:45 pg16: invalid page/page verification failed Justin Pryzby <[email protected]> 2023-10-05 17:16 ` Re: pg16: invalid page/page verification failed Matthias van de Meent <[email protected]> 2023-10-05 18:48 ` Re: pg16: invalid page/page verification failed Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox