Received: from malur.postgresql.org ([217.196.149.56]) by arkaria.postgresql.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_CBC_SHA1:256) (Exim 4.92) (envelope-from ) id 1jBLA7-00040j-Ml for pgsql-hackers@arkaria.postgresql.org; Mon, 09 Mar 2020 16:21:52 +0000 Received: from localhost ([127.0.0.1] helo=malur.postgresql.org) by malur.postgresql.org with esmtp (Exim 4.89) (envelope-from ) id 1jBLA6-0007xV-9B for pgsql-hackers@arkaria.postgresql.org; Mon, 09 Mar 2020 16:21:50 +0000 Received: from makus.postgresql.org ([2001:4800:3e1:1::229]) by malur.postgresql.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_CBC_SHA1:256) (Exim 4.89) (envelope-from ) id 1jBLA5-0007xO-9l for pgsql-hackers@lists.postgresql.org; Mon, 09 Mar 2020 16:21:49 +0000 Received: from sss.pgh.pa.us ([66.207.139.130]) by makus.postgresql.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.92) (envelope-from ) id 1jBL9q-00034h-ND for pgsql-hackers@postgresql.org; Mon, 09 Mar 2020 16:21:48 +0000 Received: from sss1.sss.pgh.pa.us (localhost [127.0.0.1]) by sss.pgh.pa.us (8.14.4/8.14.4) with ESMTP id 029GLFxm015889; Mon, 9 Mar 2020 12:21:15 -0400 From: Tom Lane To: Kyotaro Horiguchi cc: michael@paquier.xyz, alvherre@2ndquadrant.com, thomas.munro@gmail.com, tomas.vondra@2ndquadrant.com, andres@anarazel.de, a.zakirov@postgrespro.ru, ah@cybertec.at, magnus@hagander.net, robertmhaas@gmail.com, pgsql-hackers@postgresql.org Subject: Re: shared-memory based stats collector In-reply-to: <20200122.172404.1620844846813215146.horikyota.ntt@gmail.com> References: <20190927.094647.134825835.horikyota.ntt@gmail.com> <20191201021232.GO2355@paquier.xyz> <20191203.172759.631448392682474460.horikyota.ntt@gmail.com> <20200122.172404.1620844846813215146.horikyota.ntt@gmail.com> Comments: In-reply-to Kyotaro Horiguchi message dated "Wed, 22 Jan 2020 17:24:04 +0900" MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" Content-ID: <15877.1583770863.0@sss.pgh.pa.us> Date: Mon, 09 Mar 2020 12:21:15 -0400 Message-ID: <15888.1583770875@sss.pgh.pa.us> List-Id: List-Help: List-Subscribe: List-Post: List-Owner: List-Archive: Precedence: bulk ------- =_aaaaaaaaaa0 Content-Type: text/plain; charset="us-ascii" Content-ID: <15877.1583770863.1@sss.pgh.pa.us> Kyotaro Horiguchi writes: > CFbots seem unhappy with this. Rebased. It's failing to apply again, so I rebased again. I haven't read the code or done any testing beyond check-world. regards, tom lane ------- =_aaaaaaaaaa0 Content-Type: text/x-diff; name="0001-sequential-scan-for-dshash.patch"; charset="us-ascii" Content-ID: <15877.1583770863.2@sss.pgh.pa.us> Content-Description: 0001-sequential-scan-for-dshash.patch Content-Transfer-Encoding: quoted-printable diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c index 78ccf03..afaccb1 100644 --- a/src/backend/lib/dshash.c +++ b/src/backend/lib/dshash.c @@ -112,6 +112,7 @@ struct dshash_table size_t size_log2; /* log2(number of buckets) */ bool find_locked; /* Is any partition lock held by 'find'? */ bool find_exclusively_locked; /* ... exclusively? */ + bool seqscan_running;/* now under sequential scan */ }; = /* Given a pointer to an item, find the entry (user data) it holds. */ @@ -127,6 +128,10 @@ struct dshash_table #define NUM_SPLITS(size_log2) \ (size_log2 - DSHASH_NUM_PARTITIONS_LOG2) = +/* How many buckets are there in a given size? */ +#define NUM_BUCKETS(size_log2) \ + (((size_t) 1) << (size_log2)) + /* How many buckets are there in each partition at a given size? */ #define BUCKETS_PER_PARTITION(size_log2) \ (((size_t) 1) << NUM_SPLITS(size_log2)) @@ -153,6 +158,10 @@ struct dshash_table #define BUCKET_INDEX_FOR_PARTITION(partition, size_log2) \ ((partition) << NUM_SPLITS(size_log2)) = +/* Choose partition based on bucket index. */ +#define PARTITION_FOR_BUCKET_INDEX(bucket_idx, size_log2) \ + ((bucket_idx) >> NUM_SPLITS(size_log2)) + /* The head of the active bucket for a given hash value (lvalue). */ #define BUCKET_FOR_HASH(hash_table, hash) \ (hash_table->buckets[ \ @@ -228,6 +237,7 @@ dshash_create(dsa_area *area, const dshash_parameters = *params, void *arg) = hash_table->find_locked =3D false; hash_table->find_exclusively_locked =3D false; + hash_table->seqscan_running =3D false; = /* * Set up the initial array of buckets. Our initial size is the same as @@ -279,6 +289,7 @@ dshash_attach(dsa_area *area, const dshash_parameters = *params, hash_table->control =3D dsa_get_address(area, control); hash_table->find_locked =3D false; hash_table->find_exclusively_locked =3D false; + hash_table->seqscan_running =3D false; Assert(hash_table->control->magic =3D=3D DSHASH_MAGIC); = /* @@ -324,7 +335,7 @@ dshash_destroy(dshash_table *hash_table) ensure_valid_bucket_pointers(hash_table); = /* Free all the entries. */ - size =3D ((size_t) 1) << hash_table->size_log2; + size =3D NUM_BUCKETS(hash_table->size_log2); for (i =3D 0; i < size; ++i) { dsa_pointer item_pointer =3D hash_table->buckets[i]; @@ -549,9 +560,14 @@ dshash_delete_entry(dshash_table *hash_table, void *e= ntry) LW_EXCLUSIVE)); = delete_item(hash_table, item); - hash_table->find_locked =3D false; - hash_table->find_exclusively_locked =3D false; - LWLockRelease(PARTITION_LOCK(hash_table, partition)); + + /* We need to keep partition lock while sequential scan */ + if (!hash_table->seqscan_running) + { + hash_table->find_locked =3D false; + hash_table->find_exclusively_locked =3D false; + LWLockRelease(PARTITION_LOCK(hash_table, partition)); + } } = /* @@ -568,6 +584,8 @@ dshash_release_lock(dshash_table *hash_table, void *en= try) Assert(LWLockHeldByMeInMode(PARTITION_LOCK(hash_table, partition_index), hash_table->find_exclusively_locked ? LW_EXCLUSIVE : LW_SHARED)); + /* lock is under control of sequential scan */ + Assert(!hash_table->seqscan_running); = hash_table->find_locked =3D false; hash_table->find_exclusively_locked =3D false; @@ -593,6 +611,164 @@ dshash_memhash(const void *v, size_t size, void *arg= ) } = /* + * dshash_seq_init/_next/_term + * Sequentially scan trhough dshash table and return all the + * elements one by one, return NULL when no more. + * + * dshash_seq_term should be called for incomplete scans and otherwise + * shoudln't. Finished scans are cleaned up automatically. + * + * Returned elements are locked as is the case with dshash_find. However= , the + * caller must not release the lock. + * + * Same as dynanash, the caller may delete returned elements midst of a s= can. + * + * If consistent is set for dshash_seq_init, the all hash table partition= s are + * locked in the requested mode (as determined by the exclusive flag) dur= ing + * the scan. Otherwise partitions are locked in one-at-a-time way during= the + * scan. + */ +void +dshash_seq_init(dshash_seq_status *status, dshash_table *hash_table, + bool consistent, bool exclusive) +{ + /* allowed at most one scan at once */ + Assert(!hash_table->seqscan_running); + + status->hash_table =3D hash_table; + status->curbucket =3D 0; + status->nbuckets =3D 0; + status->curitem =3D NULL; + status->pnextitem =3D InvalidDsaPointer; + status->curpartition =3D -1; + status->consistent =3D consistent; + status->exclusive =3D exclusive; + hash_table->seqscan_running =3D true; + + /* + * Protect all partitions from modification if the caller wants a + * consistent result. + */ + if (consistent) + { + int i; + + for (i =3D 0; i < DSHASH_NUM_PARTITIONS; ++i) + { + Assert(!LWLockHeldByMe(PARTITION_LOCK(hash_table, i))); + + LWLockAcquire(PARTITION_LOCK(hash_table, i), + exclusive ? LW_EXCLUSIVE : LW_SHARED); + } + ensure_valid_bucket_pointers(hash_table); + } +} + +void * +dshash_seq_next(dshash_seq_status *status) +{ + dsa_pointer next_item_pointer; + + Assert(status->hash_table->seqscan_running); + if (status->curitem =3D=3D NULL) + { + int partition; + + Assert (status->curbucket =3D=3D 0); + Assert(!status->hash_table->find_locked); + + /* first shot. grab the first item. */ + if (!status->consistent) + { + partition =3D + PARTITION_FOR_BUCKET_INDEX(status->curbucket, + status->hash_table->size_log2); + LWLockAcquire(PARTITION_LOCK(status->hash_table, partition), + status->exclusive ? LW_EXCLUSIVE : LW_SHARED); + status->curpartition =3D partition; + + /* resize doesn't happen from now until seq scan ends */ + status->nbuckets =3D + NUM_BUCKETS(status->hash_table->control->size_log2); + ensure_valid_bucket_pointers(status->hash_table); + } + + next_item_pointer =3D status->hash_table->buckets[status->curbucket]; + } + else + next_item_pointer =3D status->pnextitem; + + /* Move to the next bucket if we finished the current bucket */ + while (!DsaPointerIsValid(next_item_pointer)) + { + if (++status->curbucket >=3D status->nbuckets) + { + /* all buckets have been scanned. finsih. */ + dshash_seq_term(status); + return NULL; + } + + /* Also move parititon lock if needed */ + if (!status->consistent) + { + int next_partition =3D + PARTITION_FOR_BUCKET_INDEX(status->curbucket, + status->hash_table->size_log2); + + /* Move lock along with partition for the bucket */ + if (status->curpartition !=3D next_partition) + { + /* + * Lock the next partition then release the current, not in the + * reverse order to to avoid concurrent resizing. Partitions + * are locked in the same order with resize() so dead locks + * won't happen. + */ + LWLockAcquire(PARTITION_LOCK(status->hash_table, + next_partition), + status->exclusive ? LW_EXCLUSIVE : LW_SHARED); + LWLockRelease(PARTITION_LOCK(status->hash_table, + status->curpartition)); + status->curpartition =3D next_partition; + } + } + + next_item_pointer =3D status->hash_table->buckets[status->curbucket]; + } + + status->curitem =3D + dsa_get_address(status->hash_table->area, next_item_pointer); + status->hash_table->find_locked =3D true; + status->hash_table->find_exclusively_locked =3D status->exclusive; + + /* + * The caller may delete the item. Store the next item in case of deleti= on. + */ + status->pnextitem =3D status->curitem->next; + + return ENTRY_FROM_ITEM(status->curitem); +} + +void +dshash_seq_term(dshash_seq_status *status) +{ + Assert(status->hash_table->seqscan_running); + status->hash_table->find_locked =3D false; + status->hash_table->find_exclusively_locked =3D false; + status->hash_table->seqscan_running =3D false; + + if (status->consistent) + { + int i; + + for (i =3D 0; i < DSHASH_NUM_PARTITIONS; ++i) + LWLockRelease(PARTITION_LOCK(status->hash_table, i)); + } + else if (status->curpartition >=3D 0) + LWLockRelease(PARTITION_LOCK(status->hash_table, status->curpartition))= ; +} + +/* * Print debugging information about the internal state of the hash table= to * stderr. The caller must hold no partition locks. */ @@ -673,7 +849,6 @@ delete_item(dshash_table *hash_table, dshash_table_ite= m *item) /* * Grow the hash table if necessary to the requested number of buckets. = The * requested size must be double some previously observed size. - * * Must be called without any partition lock held. */ static void diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h index b86df68..ad88f32 100644 --- a/src/include/lib/dshash.h +++ b/src/include/lib/dshash.h @@ -59,6 +59,22 @@ typedef struct dshash_parameters struct dshash_table_item; typedef struct dshash_table_item dshash_table_item; = +/* + * Sequential scan state. The detail is exposed to let users know the sto= rage + * size but it should be considered as an opaque type by callers. + */ +typedef struct dshash_seq_status +{ + dshash_table *hash_table; + int curbucket; + int nbuckets; + dshash_table_item *curitem; + dsa_pointer pnextitem; + int curpartition; + bool consistent; + bool exclusive; +} dshash_seq_status; + /* Creating, sharing and destroying from hash tables. */ extern dshash_table *dshash_create(dsa_area *area, const dshash_parameters *params, @@ -70,7 +86,6 @@ extern dshash_table *dshash_attach(dsa_area *area, extern void dshash_detach(dshash_table *hash_table); extern dshash_table_handle dshash_get_hash_table_handle(dshash_table *has= h_table); extern void dshash_destroy(dshash_table *hash_table); - /* Finding, creating, deleting entries. */ extern void *dshash_find(dshash_table *hash_table, const void *key, bool exclusive); @@ -80,6 +95,11 @@ extern bool dshash_delete_key(dshash_table *hash_table,= const void *key); extern void dshash_delete_entry(dshash_table *hash_table, void *entry); extern void dshash_release_lock(dshash_table *hash_table, void *entry); = +/* seq scan support */ +extern void dshash_seq_init(dshash_seq_status *status, dshash_table *hash= _table, + bool consistent, bool exclusive); +extern void *dshash_seq_next(dshash_seq_status *status); +extern void dshash_seq_term(dshash_seq_status *status); /* Convenience hash and compare functions wrapping memcmp and tag_hash. *= / extern int dshash_memcmp(const void *a, const void *b, size_t size, void = *arg); extern dshash_hash dshash_memhash(const void *v, size_t size, void *arg); ------- =_aaaaaaaaaa0 Content-Type: text/x-diff; name="0002-Add-conditional-lock-feature-to-dshash.patch"; charset="us-ascii" Content-ID: <15877.1583770863.3@sss.pgh.pa.us> Content-Description: 0002-Add-conditional-lock-feature-to-dshash.patch Content-Transfer-Encoding: quoted-printable diff --git a/src/backend/lib/dshash.c b/src/backend/lib/dshash.c index afaccb1..4ba6354 100644 --- a/src/backend/lib/dshash.c +++ b/src/backend/lib/dshash.c @@ -395,18 +395,50 @@ dshash_get_hash_table_handle(dshash_table *hash_tabl= e) void * dshash_find(dshash_table *hash_table, const void *key, bool exclusive) { + return dshash_find_extended(hash_table, key, exclusive, false, NULL); +} + +/* + * The version of dshash_find, which is allowed to return immediately on = lock + * failure. Lock status is set to *lock_failed in that case. + */ +void * +dshash_find_extended(dshash_table *hash_table, const void *key, + bool exclusive, bool nowait, bool *lock_acquired) +{ dshash_hash hash; size_t partition; dshash_table_item *item; = + /* + * No need to return lock resut when !nowait. Otherwise the caller may + * omit the lock result when NULL is returned. + */ + Assert(nowait || !lock_acquired); + hash =3D hash_key(hash_table, key); partition =3D PARTITION_FOR_HASH(hash); = Assert(hash_table->control->magic =3D=3D DSHASH_MAGIC); Assert(!hash_table->find_locked); = - LWLockAcquire(PARTITION_LOCK(hash_table, partition), - exclusive ? LW_EXCLUSIVE : LW_SHARED); + if (nowait) + { + if (!LWLockConditionalAcquire(PARTITION_LOCK(hash_table, partition), + exclusive ? LW_EXCLUSIVE : LW_SHARED)) + { + if (lock_acquired) + *lock_acquired =3D false; + return NULL; + } + } + else + LWLockAcquire(PARTITION_LOCK(hash_table, partition), + exclusive ? LW_EXCLUSIVE : LW_SHARED); + + if (lock_acquired) + *lock_acquired =3D true; + ensure_valid_bucket_pointers(hash_table); = /* Search the active bucket. */ @@ -442,6 +474,22 @@ dshash_find_or_insert(dshash_table *hash_table, const void *key, bool *found) { + return dshash_find_or_insert_extended(hash_table, key, found, false); +} + +/* + * The version of dshash_find_or_insert, which is allowed to return immed= iately + * on lock failure. + * + * Notes above dshash_find_extended() regarding locking and error handlin= g + * equally apply here. + */ +void * +dshash_find_or_insert_extended(dshash_table *hash_table, + const void *key, + bool *found, + bool nowait) +{ dshash_hash hash; size_t partition_index; dshash_partition *partition; @@ -455,8 +503,16 @@ dshash_find_or_insert(dshash_table *hash_table, Assert(!hash_table->find_locked); = restart: - LWLockAcquire(PARTITION_LOCK(hash_table, partition_index), - LW_EXCLUSIVE); + if (nowait) + { + if (!LWLockConditionalAcquire( + PARTITION_LOCK(hash_table, partition_index), + LW_EXCLUSIVE)) + return NULL; + } + else + LWLockAcquire(PARTITION_LOCK(hash_table, partition_index), + LW_EXCLUSIVE); ensure_valid_bucket_pointers(hash_table); = /* Search the active bucket. */ diff --git a/src/include/lib/dshash.h b/src/include/lib/dshash.h index ad88f32..a7d19c6 100644 --- a/src/include/lib/dshash.h +++ b/src/include/lib/dshash.h @@ -89,8 +89,14 @@ extern void dshash_destroy(dshash_table *hash_table); /* Finding, creating, deleting entries. */ extern void *dshash_find(dshash_table *hash_table, const void *key, bool exclusive); +extern void *dshash_find_extended(dshash_table *hash_table, const void *k= ey, + bool exclusive, bool nowait, + bool *lock_acquired); extern void *dshash_find_or_insert(dshash_table *hash_table, const void *key, bool *found); +extern void *dshash_find_or_insert_extended(dshash_table *hash_table, + const void *key, bool *found, + bool nowait); extern bool dshash_delete_key(dshash_table *hash_table, const void *key); extern void dshash_delete_entry(dshash_table *hash_table, void *entry); extern void dshash_release_lock(dshash_table *hash_table, void *entry); ------- =_aaaaaaaaaa0 Content-Type: text/x-diff; name="0003-Make-archiver-process-an-auxiliary-process.patch"; charset="us-ascii" Content-ID: <15877.1583770863.4@sss.pgh.pa.us> Content-Description: 0003-Make-archiver-process-an-auxiliary-process.patch Content-Transfer-Encoding: quoted-printable diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/boo= tstrap.c index 657b18e..c4859db 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -326,6 +326,9 @@ AuxiliaryProcessMain(int argc, char *argv[]) case StartupProcess: statmsg =3D pgstat_get_backend_desc(B_STARTUP); break; + case ArchiverProcess: + statmsg =3D pgstat_get_backend_desc(B_ARCHIVER); + break; case BgWriterProcess: statmsg =3D pgstat_get_backend_desc(B_BG_WRITER); break; @@ -451,6 +454,11 @@ AuxiliaryProcessMain(int argc, char *argv[]) StartupProcessMain(); proc_exit(1); /* should never return */ = + case ArchiverProcess: + /* don't set signals, archiver has its own agenda */ + PgArchiverMain(); + proc_exit(1); /* should never return */ + case BgWriterProcess: /* don't set signals, bgwriter has its own agenda */ BackgroundWriterMain(); diff --git a/src/backend/postmaster/pgarch.c b/src/backend/postmaster/pgar= ch.c index 3ca30ba..6441e69 100644 --- a/src/backend/postmaster/pgarch.c +++ b/src/backend/postmaster/pgarch.c @@ -78,7 +78,6 @@ * Local data * ---------- */ -static time_t last_pgarch_start_time; static time_t last_sigterm_time =3D 0; = /* @@ -95,7 +94,6 @@ static volatile sig_atomic_t ready_to_stop =3D false; static pid_t pgarch_forkexec(void); #endif = -NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]) pg_attribute_= noreturn(); static void pgarch_exit(SIGNAL_ARGS); static void pgarch_waken(SIGNAL_ARGS); static void pgarch_waken_stop(SIGNAL_ARGS); @@ -111,75 +109,6 @@ static void pgarch_archiveDone(char *xlog); * ------------------------------------------------------------ */ = -/* - * pgarch_start - * - * Called from postmaster at startup or after an existing archiver - * died. Attempt to fire up a fresh archiver process. - * - * Returns PID of child process, or 0 if fail. - * - * Note: if fail, we will be called again from the postmaster main loop. - */ -int -pgarch_start(void) -{ - time_t curtime; - pid_t pgArchPid; - - /* - * Do nothing if no archiver needed - */ - if (!XLogArchivingActive()) - return 0; - - /* - * Do nothing if too soon since last archiver start. This is a safety - * valve to protect against continuous respawn attempts if the archiver = is - * dying immediately at launch. Note that since we will be re-called fro= m - * the postmaster main loop, we will get another chance later. - */ - curtime =3D time(NULL); - if ((unsigned int) (curtime - last_pgarch_start_time) < - (unsigned int) PGARCH_RESTART_INTERVAL) - return 0; - last_pgarch_start_time =3D curtime; - -#ifdef EXEC_BACKEND - switch ((pgArchPid =3D pgarch_forkexec())) -#else - switch ((pgArchPid =3D fork_process())) -#endif - { - case -1: - ereport(LOG, - (errmsg("could not fork archiver: %m"))); - return 0; - -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); - - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); - - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); - - PgArchiverMain(0, NULL); - break; -#endif - - default: - return (int) pgArchPid; - } - - /* shouldn't get here */ - return 0; -} - /* ------------------------------------------------------------ * Local functions called by archiver follow * ------------------------------------------------------------ @@ -219,8 +148,8 @@ pgarch_forkexec(void) * The argc/argv parameters are valid only in EXEC_BACKEND case. However= , * since we don't use 'em, it hardly matters... */ -NON_EXEC_STATIC void -PgArchiverMain(int argc, char *argv[]) +void +PgArchiverMain(void) { /* * Ignore all signals usually bound to some action in the postmaster, @@ -252,8 +181,27 @@ PgArchiverMain(int argc, char *argv[]) static void pgarch_exit(SIGNAL_ARGS) { - /* SIGQUIT means curl up and die ... */ - exit(1); + PG_SETMASK(&BlockSig); + + /* + * We DO NOT want to run proc_exit() callbacks -- we're here because + * shared memory may be corrupted, so we don't want to try to clean up o= ur + * transaction. Just nail the windows shut and get out of town. Now th= at + * there's an atexit callback to prevent third-party code from breaking + * things by calling exit() directly, we have to reset the callbacks + * explicitly to make this work as intended. + */ + on_exit_reset(); + + /* + * Note we do exit(2) not exit(0). This is to force the postmaster into= a + * system reset cycle if some idiot DBA sends a manual SIGQUIT to a rand= om + * process. This is necessary precisely because we don't clean up our + * shared memory state. (The "dead man switch" mechanism in pmsignal.c + * should ensure the postmaster sees this as a crash, too, but no harm i= n + * being doubly sure.) + */ + exit(2); } = /* SIGUSR1 signal handler for archiver process */ diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgst= at.c index 462b4d7..0f5ba82 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -2930,6 +2930,9 @@ pgstat_bestart(void) case StartupProcess: lbeentry.st_backendType =3D B_STARTUP; break; + case ArchiverProcess: + beentry->st_backendType =3D B_ARCHIVER; + break; case BgWriterProcess: lbeentry.st_backendType =3D B_BG_WRITER; break; @@ -4276,6 +4279,9 @@ pgstat_get_backend_desc(BackendType backendType) = switch (backendType) { + case B_ARCHIVER: + backendDesc =3D "archiver"; + break; case B_AUTOVAC_LAUNCHER: backendDesc =3D "autovacuum launcher"; break; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/= postmaster.c index 55187eb..5df9333 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -146,7 +146,8 @@ #define BACKEND_TYPE_AUTOVAC 0x0002 /* autovacuum worker process */ #define BACKEND_TYPE_WALSND 0x0004 /* walsender process */ #define BACKEND_TYPE_BGWORKER 0x0008 /* bgworker process */ -#define BACKEND_TYPE_ALL 0x000F /* OR of all the above */ +#define BACKEND_TYPE_ARCHIVER 0x0010 /* archiver process */ +#define BACKEND_TYPE_ALL 0x001F /* OR of all the above */ = #define BACKEND_TYPE_WORKER (BACKEND_TYPE_AUTOVAC | BACKEND_TYPE_BGWORKE= R) = @@ -539,6 +540,7 @@ static void ShmemBackendArrayRemove(Backend *bn); #endif /* EXEC_BACKEND */ = #define StartupDataBase() StartChildProcess(StartupProcess) +#define StartArchiver() StartChildProcess(ArchiverProcess) #define StartBackgroundWriter() StartChildProcess(BgWriterProcess) #define StartCheckpointer() StartChildProcess(CheckpointerProcess) #define StartWalWriter() StartChildProcess(WalWriterProcess) @@ -1785,7 +1787,7 @@ ServerLoop(void) = /* If we have lost the archiver, try to start a new one. */ if (PgArchPID =3D=3D 0 && PgArchStartupAllowed()) - PgArchPID =3D pgarch_start(); + PgArchPID =3D StartArchiver(); = /* If we need to signal the autovacuum launcher, do so now */ if (avlauncher_needs_signal) @@ -3050,7 +3052,7 @@ reaper(SIGNAL_ARGS) if (!IsBinaryUpgrade && AutoVacuumingActive() && AutoVacPID =3D=3D 0) AutoVacPID =3D StartAutoVacLauncher(); if (PgArchStartupAllowed() && PgArchPID =3D=3D 0) - PgArchPID =3D pgarch_start(); + PgArchPID =3D StartArchiver(); if (PgStatPID =3D=3D 0) PgStatPID =3D pgstat_start(); = @@ -3185,20 +3187,16 @@ reaper(SIGNAL_ARGS) } = /* - * Was it the archiver? If so, just try to start a new one; no need - * to force reset of the rest of the system. (If fail, we'll try - * again in future cycles of the main loop.). Unless we were waiting - * for it to shut down; don't restart it in that case, and - * PostmasterStateMachine() will advance to the next shutdown step. + * Was it the archiver? Normal exit can be ignored; we'll start a new + * one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. */ if (pid =3D=3D PgArchPID) { PgArchPID =3D 0; if (!EXIT_STATUS_0(exitstatus)) - LogChildExit(LOG, _("archiver process"), - pid, exitstatus); - if (PgArchStartupAllowed()) - PgArchPID =3D pgarch_start(); + HandleChildCrash(pid, exitstatus, + _("archiver process")); continue; } = @@ -3446,7 +3444,7 @@ CleanupBackend(int pid, = /* * HandleChildCrash -- cleanup after failed backend, bgwriter, checkpoint= er, - * walwriter, autovacuum, or background worker. + * walwriter, autovacuum, archiver or background worker. * * The objectives here are to clean up our local state about the child * process, and to signal all other remaining children to quickdie. @@ -3651,6 +3649,18 @@ HandleChildCrash(int pid, int exitstatus, const cha= r *procname) signal_child(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT)); } = + /* Take care of the archiver too */ + if (pid =3D=3D PgArchPID) + PgArchPID =3D 0; + else if (PgArchPID !=3D 0 && take_action) + { + ereport(DEBUG2, + (errmsg_internal("sending %s to process %d", + (SendStop ? "SIGSTOP" : "SIGQUIT"), + (int) PgArchPID))); + signal_child(PgArchPID, (SendStop ? SIGSTOP : SIGQUIT)); + } + /* * Force a power-cycle of the pgarch process too. (This isn't absolutel= y * necessary, but it seems like a good idea for robustness, and it @@ -3923,6 +3933,7 @@ PostmasterStateMachine(void) Assert(CheckpointerPID =3D=3D 0); Assert(WalWriterPID =3D=3D 0); Assert(AutoVacPID =3D=3D 0); + Assert(PgArchPID =3D=3D 0); /* syslogger is not considered here */ pmState =3D PM_NO_CHILDREN; } @@ -5207,7 +5218,7 @@ sigusr1_handler(SIGNAL_ARGS) */ Assert(PgArchPID =3D=3D 0); if (XLogArchivingAlways()) - PgArchPID =3D pgarch_start(); + PgArchPID =3D StartArchiver(); = /* * If we aren't planning to enter hot standby mode later, treat @@ -5492,6 +5503,10 @@ StartChildProcess(AuxProcType type) ereport(LOG, (errmsg("could not fork startup process: %m"))); break; + case ArchiverProcess: + ereport(LOG, + (errmsg("could not fork archiver process: %m"))); + break; case BgWriterProcess: ereport(LOG, (errmsg("could not fork background writer process: %m"))); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f985453..ef4cb6a 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -397,6 +397,7 @@ typedef enum BootstrapProcess, StartupProcess, BgWriterProcess, + ArchiverProcess, CheckpointerProcess, WalWriterProcess, WalReceiverProcess, @@ -409,6 +410,7 @@ extern AuxProcType MyAuxProcType; #define AmBootstrapProcess() (MyAuxProcType =3D=3D BootstrapProcess) #define AmStartupProcess() (MyAuxProcType =3D=3D StartupProcess) #define AmBackgroundWriterProcess() (MyAuxProcType =3D=3D BgWriterProcess= ) +#define AmArchiverProcess() (MyAuxProcType =3D=3D ArchiverProcess) #define AmCheckpointerProcess() (MyAuxProcType =3D=3D CheckpointerProces= s) #define AmWalWriterProcess() (MyAuxProcType =3D=3D WalWriterProcess) #define AmWalReceiverProcess() (MyAuxProcType =3D=3D WalReceiverProcess) diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 7bc36c6..dfdac00 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -718,6 +718,7 @@ typedef struct PgStat_GlobalStats */ typedef enum BackendType { + B_ARCHIVER, B_AUTOVAC_LAUNCHER, B_AUTOVAC_WORKER, B_BACKEND, diff --git a/src/include/postmaster/pgarch.h b/src/include/postmaster/pgar= ch.h index b320087..e3ffc63 100644 --- a/src/include/postmaster/pgarch.h +++ b/src/include/postmaster/pgarch.h @@ -32,8 +32,6 @@ */ extern int pgarch_start(void); = -#ifdef EXEC_BACKEND -extern void PgArchiverMain(int argc, char *argv[]) pg_attribute_noreturn(= ); -#endif +extern void PgArchiverMain(void) pg_attribute_noreturn(); = #endif /* _PGARCH_H */ ------- =_aaaaaaaaaa0 Content-Type: text/x-diff; name="0004-Shared-memory-based-stats-collector.patch"; charset="us-ascii" Content-ID: <15877.1583770863.5@sss.pgh.pa.us> Content-Description: 0004-Shared-memory-based-stats-collector.patch Content-Transfer-Encoding: quoted-printable diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 987580d..11aaef5 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -53,7 +53,6 @@ postgres 15554 0.0 0.0 57536 1184 ? Ss 18:= 02 0:00 postgres: back postgres 15555 0.0 0.0 57536 916 ? Ss 18:02 0:00 postgre= s: checkpointer postgres 15556 0.0 0.0 57536 916 ? Ss 18:02 0:00 postgre= s: walwriter postgres 15557 0.0 0.0 58504 2244 ? Ss 18:02 0:00 postgre= s: autovacuum launcher -postgres 15558 0.0 0.0 17512 1068 ? Ss 18:02 0:00 postgre= s: stats collector postgres 15582 0.0 0.0 58772 3080 ? Ss 18:04 0:00 postgre= s: joe runbug 127.0.0.1 idle postgres 15606 0.0 0.0 58772 3052 ? Ss 18:07 0:00 postgre= s: tgl regression [local] SELECT waiting postgres 15610 0.0 0.0 58772 3056 ? Ss 18:07 0:00 postgre= s: tgl regression [local] idle in transaction @@ -65,9 +64,8 @@ postgres 15610 0.0 0.0 58772 3056 ? Ss 18:= 07 0:00 postgres: tgl master server process. The command arguments shown for it are the same ones used when it was launched. The next fi= ve processes are background worker processes automatically launched by th= e - master process. (The stats collector process will not = be present - if you have set the system not to start the statistics collector; like= wise - the autovacuum launcher process can be disabled.) + master process. (The autovacuum launcher process will = not + be present if you have set the system not to start it.) Each of the remaining processes is a server process handling one client connection. Each su= ch process sets its command line display in the form diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/= autovacuum.c index e3a43d3..809ad51 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -1956,15 +1956,15 @@ do_autovacuum(void) ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(AutovacMemCxt); = + /* Start a transaction so our commands have one to play into. */ + StartTransactionCommand(); + /* * may be NULL if we couldn't find an entry (only happens if we are * forcing a vacuum for anti-wrap purposes). */ dbentry =3D pgstat_fetch_stat_dbentry(MyDatabaseId); = - /* Start a transaction so our commands have one to play into. */ - StartTransactionCommand(); - /* * Clean up any dead statistics collector entries for this DB. We always * want to do this exactly once per DB-processing cycle, even if we find @@ -2748,12 +2748,10 @@ get_pgstat_tabentry_relid(Oid relid, bool isshared= , PgStat_StatDBEntry *shared, if (isshared) { if (PointerIsValid(shared)) - tabentry =3D hash_search(shared->tables, &relid, - HASH_FIND, NULL); + tabentry =3D pgstat_fetch_stat_tabentry_extended(shared, relid); } else if (PointerIsValid(dbentry)) - tabentry =3D hash_search(dbentry->tables, &relid, - HASH_FIND, NULL); + tabentry =3D pgstat_fetch_stat_tabentry_extended(dbentry, relid); = return tabentry; } diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgst= at.c index 0f5ba82..8ea8e52 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -1,15 +1,23 @@ /* ---------- * pgstat.c * - * All the statistics collector stuff hacked up in one big, ugly file. + * Statistics collector facility. * - * TODO: - Separate collector, postmaster and backend stuff - * into different files. + * Collects per-table and per-function usage statistics of all backends = on + * shared memory. pg_count_*() and friends are the interface to locally = store + * backend activities during a transaction. Then pgstat_flush_stat() is = called + * at the end of a transaction to pulish the local stats on shared memor= y. * - * - Add some automatic call for pgstat vacuuming. + * To avoid congestion on the shared memory, we update shared stats no m= ore + * often than intervals of PGSTAT_STAT_MIN_INTERVAL(500ms). In the case = where + * all the local numbers cannot be flushed immediately, we postpone upda= tes + * and try the next chance after the interval of + * PGSTAT_STAT_RETRY_INTERVAL(100ms), but we don't wait for no longer th= an + * PGSTAT_STAT_MAX_INTERVAL(1000ms). * - * - Add a pgstat config column to pg_database, so this - * entire thing can be enabled/disabled on a per db basis. + * The first process that uses stats collector creates the area then loa= d the + * stored stats file if any, and the last process at shutdown writes the + * shared stats to the file then destroy the area before exit. * * Copyright (c) 2001-2020, PostgreSQL Global Development Group * @@ -19,18 +27,6 @@ #include "postgres.h" = #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef HAVE_SYS_SELECT_H -#include -#endif = #include "access/heapam.h" #include "access/htup_details.h" @@ -40,67 +36,42 @@ #include "access/xact.h" #include "catalog/pg_database.h" #include "catalog/pg_proc.h" -#include "common/ip.h" #include "libpq/libpq.h" -#include "libpq/pqsignal.h" -#include "mb/pg_wchar.h" #include "miscadmin.h" -#include "pg_trace.h" #include "pgstat.h" #include "postmaster/autovacuum.h" #include "postmaster/fork_process.h" #include "postmaster/interrupt.h" #include "postmaster/postmaster.h" #include "replication/walsender.h" -#include "storage/backendid.h" -#include "storage/dsm.h" -#include "storage/fd.h" #include "storage/ipc.h" -#include "storage/latch.h" #include "storage/lmgr.h" -#include "storage/pg_shmem.h" +#include "storage/proc.h" #include "storage/procsignal.h" #include "storage/sinvaladt.h" #include "utils/ascii.h" #include "utils/guc.h" #include "utils/memutils.h" -#include "utils/ps_status.h" -#include "utils/rel.h" +#include "utils/probes.h" #include "utils/snapmgr.h" -#include "utils/timestamp.h" = /* ---------- * Timer definitions. * ---------- */ -#define PGSTAT_STAT_INTERVAL 500 /* Minimum time between stats file - * updates; in milliseconds. */ - -#define PGSTAT_RETRY_DELAY 10 /* How long to wait between checks for a - * new file; in milliseconds. */ - -#define PGSTAT_MAX_WAIT_TIME 10000 /* Maximum time to wait for a stats - * file update; in milliseconds. */ - -#define PGSTAT_INQ_INTERVAL 640 /* How often to ping the collector for a - * new file; in milliseconds. */ +#define PGSTAT_STAT_MIN_INTERVAL 500 /* Minimum interval of stats data + * updates; in milliseconds. */ = -#define PGSTAT_RESTART_INTERVAL 60 /* How often to attempt to restart a - * failed statistics collector; in - * seconds. */ - -#define PGSTAT_POLL_LOOP_COUNT (PGSTAT_MAX_WAIT_TIME / PGSTAT_RETRY_DELAY= ) -#define PGSTAT_INQ_LOOP_COUNT (PGSTAT_INQ_INTERVAL / PGSTAT_RETRY_DELAY) - -/* Minimum receive buffer size for the collector's socket. */ -#define PGSTAT_MIN_RCVBUF (100 * 1024) +#define PGSTAT_STAT_RETRY_INTERVAL 100 /* Retry interval after + * PGSTAT_MIN_INTERVAL*/ = +#define PGSTAT_STAT_MAX_INTERVAL 1000 /* Longest interval of stats data + * updates; in milliseconds. */ = /* ---------- * The initial size hints for the hash tables used in the collector. * ---------- */ -#define PGSTAT_DB_HASH_SIZE 16 #define PGSTAT_TAB_HASH_SIZE 512 #define PGSTAT_FUNCTION_HASH_SIZE 512 = @@ -116,6 +87,19 @@ */ #define NumBackendStatSlots (MaxBackends + NUM_AUXPROCTYPES) = +/* + * Operation mode and return code of pgstat_get_db_entry. + */ +#define PGSTAT_SHARED 0 +#define PGSTAT_EXCLUSIVE 1 +#define PGSTAT_NOWAIT 2 + +typedef enum PgStat_TableLookupResult +{ + NOT_FOUND, + FOUND, + LOCK_FAILED +} PgStat_TableLookupResult; = /* ---------- * GUC parameters @@ -131,31 +115,63 @@ int pgstat_track_activity_query_size =3D 1024; * ---------- */ char *pgstat_stat_directory =3D NULL; + +/* No longer used, but will be removed with GUC */ char *pgstat_stat_filename =3D NULL; char *pgstat_stat_tmpname =3D NULL; = -/* - * BgWriter global statistics counters (unused in other processes). - * Stored directly in a stats message structure so it can be sent - * without needing to copy things around. We assume this inits to zeroes= . - */ -PgStat_MsgBgWriter BgWriterStats; - -/* ---------- - * Local data - * ---------- - */ -NON_EXEC_STATIC pgsocket pgStatSock =3D PGINVALID_SOCKET; - -static struct sockaddr_storage pgStatAddr; +#define StatsLock (&StatsShmem->StatsMainLock) = -static time_t last_pgstat_start_time; +/* Shared stats bootstrap information */ +typedef struct StatsShmemStruct +{ + LWLock StatsMainLock; /* lock to protect this struct */ + dsa_handle stats_dsa_handle; /* DSA handle for stats data */ + dshash_table_handle db_hash_handle; + dsa_pointer global_stats; + dsa_pointer archiver_stats; + int refcount; +} StatsShmemStruct; = -static bool pgStatRunningInCollector =3D false; +/* + * BgWriter global statistics counters. The name cntains a remnant from t= he + * time when the stats collector was a dedicate process, which used socke= ts to + * send it. + */ +PgStat_MsgBgWriter BgWriterStats =3D {0}; + +/* backend-lifetime storages */ +static StatsShmemStruct * StatsShmem =3D NULL; +static dsa_area *area =3D NULL; +static dshash_table *pgStatDBHash =3D NULL; + + +/* parameter for shared hashes */ +static const dshash_parameters dsh_dbparams =3D { + sizeof(Oid), + SHARED_DBENT_SIZE, + dshash_memcmp, + dshash_memhash, + LWTRANCHE_STATS +}; +static const dshash_parameters dsh_tblparams =3D { + sizeof(Oid), + sizeof(PgStat_StatTabEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_STATS +}; +static const dshash_parameters dsh_funcparams =3D { + sizeof(Oid), + sizeof(PgStat_StatFuncEntry), + dshash_memcmp, + dshash_memhash, + LWTRANCHE_STATS +}; = /* * Structures in which backends store per-table info that's waiting to be - * sent to the collector. + * written to shared memory. * * NOTE: once allocated, TabStatusArray structures are never moved or del= eted * for the life of the backend. Also, we zero out the t_id fields of the @@ -190,8 +206,8 @@ typedef struct TabStatHashEntry static HTAB *pgStatTabHash =3D NULL; = /* - * Backends store per-function info that's waiting to be sent to the coll= ector - * in this hash table (indexed by function OID). + * Backends store per-function info that's waiting to be flushed out to s= hared + * memory in this hash table (indexed by function OID). */ static HTAB *pgStatFunctions =3D NULL; = @@ -201,6 +217,71 @@ static HTAB *pgStatFunctions =3D NULL; */ static bool have_function_stats =3D false; = +/* common header of snapshot entry in reader snapshot hash */ +typedef struct PgStat_snapshot +{ + Oid key; + bool negative; + void *body; /* end of header part: to keep alignment */ +} PgStat_snapshot; + +/* context struct for snapshot_statentry */ +typedef struct pgstat_snapshot_param +{ + char *hash_name; /* name of the snapshot hash */ + int hash_entsize; /* element size of hash entry */ + dshash_table_handle dsh_handle; /* dsh handle to attach */ + const dshash_parameters *dsh_params;/* dshash params */ + HTAB **hash; /* points to variable to hold hash */ + dshash_table **dshash; /* ditto for dshash */ +} pgstat_snapshot_param; + +/* + * Backends store various database-wide info that's waiting to be flushed= out + * to shared memory in these variables. + * + * checksum_failures is the exception in that it is cluster-wide value. + */ +typedef struct BackendDBStats +{ + int n_conflict_tablespace; + int n_conflict_lock; + int n_conflict_snapshot; + int n_conflict_bufferpin; + int n_conflict_startup_deadlock; + int n_deadlocks; + size_t n_tmpfiles; + size_t tmpfilesize; + HTAB *checksum_failures; +} BackendDBStats; + +/* Hash entry struct for checksum_failures above */ +typedef struct ChecksumFailureEnt +{ + Oid dboid; + int count; +} ChecksumFailureEnt; + +static BackendDBStats BeDBStats =3D {0}; + +/* macros to check BeDBStats at once */ + +/* checks if there is any conflict waiting to be flushed out */ +#define HAVE_PENDING_CONFLICTS() \ + (BeDBStats.n_conflict_tablespace > 0 || \ + BeDBStats.n_conflict_lock > 0 || \ + BeDBStats.n_conflict_bufferpin > 0 || \ + BeDBStats.n_conflict_startup_deadlock > 0) + +/* checks if there is any database-wide waiting to be flushed out */ +#define HAVE_PENDING_DBSTATS() \ + (HAVE_PENDING_CONFLICTS() || \ + BeDBStats.n_deadlocks > 0 || \ + BeDBStats.n_tmpfiles > 0 || \ + /* no need to check tmpfilesize */ \ + BeDBStats.checksum_failures !=3D NULL) + + /* * Tuple insertion/deletion counts for an open transaction can't be propa= gated * into PgStat_TableStatus counters until we know if it is going to commi= t @@ -236,11 +317,11 @@ typedef struct TwoPhasePgStatRecord bool t_truncated; /* was the relation truncated? */ } TwoPhasePgStatRecord; = -/* - * Info about current "snapshot" of stats file - */ +/* Variables for backend status snapshot */ static MemoryContext pgStatLocalContext =3D NULL; -static HTAB *pgStatDBHash =3D NULL; +static MemoryContext pgStatSnapshotContext =3D NULL; +static HTAB *pgStatLocalHash =3D NULL; +static bool clear_snapshot =3D false; = /* Status for backends including auxiliary */ static LocalPgBackendStatus *localBackendStatusTable =3D NULL; @@ -249,19 +330,35 @@ static LocalPgBackendStatus *localBackendStatusTable= =3D NULL; static int localNumBackends =3D 0; = /* - * Cluster wide statistics, kept in the stats collector. - * Contains statistics that are not collected per database - * or per table. + * Struct for context for pgstat_flush_* functions + * + * To avoid repeated attach/detch of the same dshash, dshashes once attac= hed is + * stored in this structure and moved around multiple calls of multiple + * functions. generation here means the value returned by pin_hashes(). */ -static PgStat_ArchiverStats archiverStats; -static PgStat_GlobalStats globalStats; +typedef struct pgstat_flush_stat_context +{ + int shgeneration; /* "generation" of shdb_tabhash below */ + PgStat_StatDBEntry *shdbentry; /* dbentry for shared tables (oid =3D 0) = */ + dshash_table *shdb_tabhash; /* tabentry dshash of shared tables */ + + int mygeneration; /* "generation" of mydb_tabhash below */ + PgStat_StatDBEntry *mydbentry; /* dbengry for my database */ + dshash_table *mydb_tabhash; /* tabentry dshash of my database */ +} pgstat_flush_stat_context; = /* - * List of OIDs of databases we need to write out. If an entry is Invali= dOid, - * it means to write only the shared-catalog stats ("DB 0"); otherwise, w= e - * will write both that DB's data and the shared stats. + * Cluster wide statistics. + * + * Contains statistics that are collected not per database nor per table + * basis. shared_* points to shared memroy and snapshot_* are backend + * snapshots. Their validity is indicated by global_snapshot_is_valid. */ -static List *pending_write_requests =3D NIL; +static bool global_snapshot_is_valid =3D false; +static PgStat_ArchiverStats *shared_archiverStats; +static PgStat_ArchiverStats snapshot_archiverStats; +static PgStat_GlobalStats *shared_globalStats; +static PgStat_GlobalStats snapshot_globalStats; = /* * Total time charged to functions so far in the current backend. @@ -275,33 +372,40 @@ static instr_time total_func_time; * Local function forward declarations * ---------- */ -#ifdef EXEC_BACKEND -static pid_t pgstat_forkexec(void); -#endif - -NON_EXEC_STATIC void PgstatCollectorMain(int argc, char *argv[]) pg_attri= bute_noreturn(); static void pgstat_beshutdown_hook(int code, Datum arg); - -static PgStat_StatDBEntry *pgstat_get_db_entry(Oid databaseid, bool creat= e); -static PgStat_StatTabEntry *pgstat_get_tab_entry(PgStat_StatDBEntry *dben= try, +static PgStat_StatDBEntry *pgstat_get_db_entry(Oid databaseid, int op, + PgStat_TableLookupResult *status); +static PgStat_StatTabEntry *pgstat_get_tab_entry(dshash_table *table, Oid tableoid, bool create); -static void pgstat_write_statsfiles(bool permanent, bool allDbs); -static void pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool p= ermanent); -static HTAB *pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep= ); -static void pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB = *funchash, bool permanent); -static void backend_read_statsfile(void); +static void pgstat_write_pgStatDBHashfile(PgStat_StatDBEntry *dbentry); +static void pgstat_read_pgStatDBHashfile(PgStat_StatDBEntry *dbentry); static void pgstat_read_current_status(void); - -static bool pgstat_write_statsfile_needed(void); -static bool pgstat_db_requested(Oid databaseid); - -static void pgstat_send_tabstat(PgStat_MsgTabstat *tsmsg); -static void pgstat_send_funcstats(void); +static bool pgstat_flush_stat(pgstat_flush_stat_context *cxt, bool nowait= ); +static bool pgstat_flush_tabstat(pgstat_flush_stat_context *cxt, bool now= ait, + PgStat_TableStatus *entry); +static bool pgstat_flush_funcstats(pgstat_flush_stat_context *cxt, bool n= owait); +static bool pgstat_flush_dbstats(pgstat_flush_stat_context *cxt, bool now= ait); +static bool pgstat_update_tabentry(dshash_table *tabhash, + PgStat_TableStatus *stat, bool nowait); +static void pgstat_update_dbentry(PgStat_StatDBEntry *dbentry, + PgStat_TableStatus *stat); static HTAB *pgstat_collect_oids(Oid catalogid, AttrNumber anum_oid); = +static void pgstat_remove_useless_entries(const dshash_table_handle dshha= ndle, + const dshash_parameters *dshparams, + HTAB *oidtab); static PgStat_TableStatus *get_tabstat_entry(Oid rel_id, bool isshared); = static void pgstat_setup_memcxt(void); +static void pgstat_flush_recovery_conflict(PgStat_StatDBEntry *dbentry); +static void pgstat_flush_deadlock(PgStat_StatDBEntry *dbentry); +static void pgstat_flush_checksum_failure(PgStat_StatDBEntry *dbentry); +static void pgstat_flush_tempfile(PgStat_StatDBEntry *dbentry); +static HTAB *create_tabstat_hash(void); +static PgStat_SubXactStatus *get_tabstat_stack_level(int nest_level); +static void add_tabstat_xact_level(PgStat_TableStatus *pgstat_info, int n= est_level); +static PgStat_StatFuncEntry *pgstat_fetch_stat_funcentry_extended(PgStat_= StatDBEntry *dbent, Oid funcid); +static void pgstat_snapshot_global_stats(void); = static const char *pgstat_get_wait_activity(WaitEventActivity w); static const char *pgstat_get_wait_client(WaitEventClient w); @@ -309,560 +413,497 @@ static const char *pgstat_get_wait_ipc(WaitEventIP= C w); static const char *pgstat_get_wait_timeout(WaitEventTimeout w); static const char *pgstat_get_wait_io(WaitEventIO w); = -static void pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype); -static void pgstat_send(void *msg, int len); - -static void pgstat_recv_inquiry(PgStat_MsgInquiry *msg, int len); -static void pgstat_recv_tabstat(PgStat_MsgTabstat *msg, int len); -static void pgstat_recv_tabpurge(PgStat_MsgTabpurge *msg, int len); -static void pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len); -static void pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len= ); -static void pgstat_recv_resetsharedcounter(PgStat_MsgResetsharedcounter *= msg, int len); -static void pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *= msg, int len); -static void pgstat_recv_autovac(PgStat_MsgAutovacStart *msg, int len); -static void pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len); -static void pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len); -static void pgstat_recv_archiver(PgStat_MsgArchiver *msg, int len); -static void pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len); -static void pgstat_recv_funcstat(PgStat_MsgFuncstat *msg, int len); -static void pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len); -static void pgstat_recv_recoveryconflict(PgStat_MsgRecoveryConflict *msg,= int len); -static void pgstat_recv_deadlock(PgStat_MsgDeadlock *msg, int len); -static void pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, = int len); -static void pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len); +/* ------------------------------------------------------------ + * Local support functions follow + * ------------------------------------------------------------ + */ +static int pin_hashes(PgStat_StatDBEntry *dbentry); +static void unpin_hashes(PgStat_StatDBEntry *dbentry, int generation); +static dshash_table *attach_table_hash(PgStat_StatDBEntry *dbent, int gen= ); +static dshash_table *attach_function_hash(PgStat_StatDBEntry *dbent, int = gen); +static void reset_dbentry_counters(PgStat_StatDBEntry *dbentry); = /* ------------------------------------------------------------ * Public functions called from postmaster follow * ------------------------------------------------------------ */ = -/* ---------- - * pgstat_init() - - * - * Called from postmaster at startup. Create the resources required - * by the statistics collector process. If unable to do so, do not - * fail --- better to let the postmaster start with stats collection - * disabled. - * ---------- +/* + * StatsShmemSize + * Compute space needed for stats collector's shared memory + */ +Size +StatsShmemSize(void) +{ + return sizeof(StatsShmemStruct); +} + +/* + * StatsShmemInit - initialize during shared-memory creation */ void -pgstat_init(void) +StatsShmemInit(void) { - ACCEPT_TYPE_ARG3 alen; - struct addrinfo *addrs =3D NULL, - *addr, - hints; - int ret; - fd_set rset; - struct timeval tv; - char test_byte; - int sel_res; - int tries =3D 0; - -#define TESTBYTEVAL ((char) 199) + bool found; = - /* - * This static assertion verifies that we didn't mess up the calculation= s - * involved in selecting maximum payload sizes for our UDP messages. - * Because the only consequence of overrunning PGSTAT_MAX_MSG_SIZE would - * be silent performance loss from fragmentation, it seems worth having = a - * compile-time cross-check that we didn't. - */ - StaticAssertStmt(sizeof(PgStat_Msg) <=3D PGSTAT_MAX_MSG_SIZE, - "maximum stats message size exceeds PGSTAT_MAX_MSG_SIZE"); + StatsShmem =3D (StatsShmemStruct *) + ShmemInitStruct("Stats area", StatsShmemSize(), + &found); = - /* - * Create the UDP socket for sending and receiving statistic messages - */ - hints.ai_flags =3D AI_PASSIVE; - hints.ai_family =3D AF_UNSPEC; - hints.ai_socktype =3D SOCK_DGRAM; - hints.ai_protocol =3D 0; - hints.ai_addrlen =3D 0; - hints.ai_addr =3D NULL; - hints.ai_canonname =3D NULL; - hints.ai_next =3D NULL; - ret =3D pg_getaddrinfo_all("localhost", NULL, &hints, &addrs); - if (ret || !addrs) + if (!IsUnderPostmaster) { - ereport(LOG, - (errmsg("could not resolve \"localhost\": %s", - gai_strerror(ret)))); - goto startup_failed; - } + Assert(!found); = - /* - * On some platforms, pg_getaddrinfo_all() may return multiple addresses - * only one of which will actually work (eg, both IPv6 and IPv4 addresse= s - * when kernel will reject IPv6). Worse, the failure may occur at the - * bind() or perhaps even connect() stage. So we must loop through the - * results till we find a working combination. We will generate LOG - * messages, but no error, for bogus combinations. - */ - for (addr =3D addrs; addr; addr =3D addr->ai_next) - { -#ifdef HAVE_UNIX_SOCKETS - /* Ignore AF_UNIX sockets, if any are returned. */ - if (addr->ai_family =3D=3D AF_UNIX) - continue; -#endif + StatsShmem->stats_dsa_handle =3D DSM_HANDLE_INVALID; + } = - if (++tries > 1) - ereport(LOG, - (errmsg("trying another address for the statistics collector"))); + LWLockInitialize(StatsLock, LWTRANCHE_STATS); +} = - /* - * Create the socket. - */ - if ((pgStatSock =3D socket(addr->ai_family, SOCK_DGRAM, 0)) =3D=3D PGIN= VALID_SOCKET) - { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not create socket for statistics collector: %m"))); - continue; - } +/* ---------- + * pgstat_attach_shared_stats() - + * + * Attach shared or create stats memory. + * --------- + */ +static void +pgstat_attach_shared_stats(void) +{ + MemoryContext oldcontext; = - /* - * Bind it to a kernel assigned port on localhost and get the assigned - * port via getsockname(). - */ - if (bind(pgStatSock, addr->ai_addr, addr->ai_addrlen) < 0) - { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not bind socket for statistics collector: %m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + /* + * Don't use dsm under postmaster, or when not tracking counts. + */ + if (!pgstat_track_counts || !IsUnderPostmaster) + return; = - alen =3D sizeof(pgStatAddr); - if (getsockname(pgStatSock, (struct sockaddr *) &pgStatAddr, &alen) < 0= ) - { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not get address of socket for statistics collector: %= m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + pgstat_setup_memcxt(); = - /* - * Connect the socket to its own address. This saves a few cycles by - * not having to respecify the target address on every send. This also - * provides a kernel-level check that only packets from this same - * address will be received. - */ - if (connect(pgStatSock, (struct sockaddr *) &pgStatAddr, alen) < 0) - { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not connect socket for statistics collector: %m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + if (area) + return; = - /* - * Try to send and receive a one-byte test message on the socket. This - * is to catch situations where the socket can be created but will not - * actually pass data (for instance, because kernel packet filtering - * rules prevent it). - */ - test_byte =3D TESTBYTEVAL; + oldcontext =3D MemoryContextSwitchTo(TopMemoryContext); = -retry1: - if (send(pgStatSock, &test_byte, 1, 0) !=3D 1) - { - if (errno =3D=3D EINTR) - goto retry1; /* if interrupted, just retry */ - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not send test message on socket for statistics collec= tor: %m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + LWLockAcquire(StatsLock, LW_EXCLUSIVE); = - /* - * There could possibly be a little delay before the message can be - * received. We arbitrarily allow up to half a second before deciding - * it's broken. - */ - for (;;) /* need a loop to handle EINTR */ - { - FD_ZERO(&rset); - FD_SET(pgStatSock, &rset); + if (StatsShmem->refcount > 0) + StatsShmem->refcount++; + else + { + /* Need to create shared memory area and load saved stats if any. */ + Assert(StatsShmem->stats_dsa_handle =3D=3D DSM_HANDLE_INVALID); = - tv.tv_sec =3D 0; - tv.tv_usec =3D 500000; - sel_res =3D select(pgStatSock + 1, &rset, NULL, NULL, &tv); - if (sel_res >=3D 0 || errno !=3D EINTR) - break; - } - if (sel_res < 0) - { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("select() failed in statistics collector: %m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } - if (sel_res =3D=3D 0 || !FD_ISSET(pgStatSock, &rset)) - { - /* - * This is the case we actually think is likely, so take pains to - * give a specific message for it. - * - * errno will not be set meaningfully here, so don't use it. - */ - ereport(LOG, - (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("test message did not get through on socket for statistics c= ollector"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + /* Initialize shared memory area */ + area =3D dsa_create(LWTRANCHE_STATS); + pgStatDBHash =3D dshash_create(area, &dsh_dbparams, 0); = - test_byte++; /* just make sure variable is changed */ + StatsShmem->stats_dsa_handle =3D dsa_get_handle(area); + StatsShmem->global_stats =3D + dsa_allocate0(area, sizeof(PgStat_GlobalStats)); + StatsShmem->archiver_stats =3D + dsa_allocate0(area, sizeof(PgStat_ArchiverStats)); + StatsShmem->db_hash_handle =3D dshash_get_hash_table_handle(pgStatDBHas= h); = -retry2: - if (recv(pgStatSock, &test_byte, 1, 0) !=3D 1) - { - if (errno =3D=3D EINTR) - goto retry2; /* if interrupted, just retry */ - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not receive test message on socket for statistics col= lector: %m"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + shared_globalStats =3D (PgStat_GlobalStats *) + dsa_get_address(area, StatsShmem->global_stats); + shared_archiverStats =3D (PgStat_ArchiverStats *) + dsa_get_address(area, StatsShmem->archiver_stats); = - if (test_byte !=3D TESTBYTEVAL) /* strictly paranoia ... */ - { - ereport(LOG, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("incorrect test message transmission on socket for statistic= s collector"))); - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; - continue; - } + /* Load saved data if any. */ + pgstat_read_statsfiles(); = - /* If we get here, we have a working socket */ - break; + StatsShmem->refcount =3D 1; } = - /* Did we find a working address? */ - if (!addr || pgStatSock =3D=3D PGINVALID_SOCKET) - goto startup_failed; + LWLockRelease(StatsLock); = /* - * Set the socket to non-blocking IO. This ensures that if the collecto= r - * falls behind, statistics messages will be discarded; backends won't - * block waiting to send messages to the collector. + * If we're not the first process, attach existing shared stats area + * outside StatsLock. */ - if (!pg_set_noblock(pgStatSock)) + if (!area) { - ereport(LOG, - (errcode_for_socket_access(), - errmsg("could not set statistics collector socket to nonblocking mod= e: %m"))); - goto startup_failed; + /* Shared area already exists. Just attach it. */ + area =3D dsa_attach(StatsShmem->stats_dsa_handle); + pgStatDBHash =3D dshash_attach(area, &dsh_dbparams, + StatsShmem->db_hash_handle, 0); + + /* Setup local variables */ + pgStatLocalHash =3D NULL; + shared_globalStats =3D (PgStat_GlobalStats *) + dsa_get_address(area, StatsShmem->global_stats); + shared_archiverStats =3D (PgStat_ArchiverStats *) + dsa_get_address(area, StatsShmem->archiver_stats); } = - /* - * Try to ensure that the socket's receive buffer is at least - * PGSTAT_MIN_RCVBUF bytes, so that it won't easily overflow and lose - * data. Use of UDP protocol means that we are willing to lose data und= er - * heavy load, but we don't want it to happen just because of ridiculous= ly - * small default buffer sizes (such as 8KB on older Windows versions). - */ - { - int old_rcvbuf; - int new_rcvbuf; - ACCEPT_TYPE_ARG3 rcvbufsize =3D sizeof(old_rcvbuf); + MemoryContextSwitchTo(oldcontext); = - if (getsockopt(pgStatSock, SOL_SOCKET, SO_RCVBUF, - (char *) &old_rcvbuf, &rcvbufsize) < 0) - { - elog(LOG, "getsockopt(SO_RCVBUF) failed: %m"); - /* if we can't get existing size, always try to set it */ - old_rcvbuf =3D 0; - } + /* don't detach automatically */ + dsa_pin_mapping(area); + global_snapshot_is_valid =3D false; +} = - new_rcvbuf =3D PGSTAT_MIN_RCVBUF; - if (old_rcvbuf < new_rcvbuf) - { - if (setsockopt(pgStatSock, SOL_SOCKET, SO_RCVBUF, - (char *) &new_rcvbuf, sizeof(new_rcvbuf)) < 0) - elog(LOG, "setsockopt(SO_RCVBUF) failed: %m"); - } +/* ---------- + * pgstat_detach_shared_stats() - + * + * Detach shared stats. Write out to file if we're the last process and t= old + * to do so. + * ---------- + */ +static void +pgstat_detach_shared_stats(bool write_stats) +{ + /* immediately return if useless */ + if (!area || !IsUnderPostmaster) + return; + + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + + /* write out the shared stats to file if needed */ + if (--StatsShmem->refcount < 1) + { + if (write_stats) + pgstat_write_statsfiles(); + + /* We're the last process. Invalidate the dsa area handle. */ + StatsShmem->stats_dsa_handle =3D DSM_HANDLE_INVALID; } = - pg_freeaddrinfo_all(hints.ai_family, addrs); + LWLockRelease(StatsLock); = - /* Now that we have a long-lived socket, tell fd.c about it. */ - ReserveExternalFD(); + /* + * Detach the area. Automatically destroyed when the last process detach= ed + * it. + */ + dsa_detach(area); = - return; + area =3D NULL; + pgStatDBHash =3D NULL; + shared_globalStats =3D NULL; + shared_archiverStats =3D NULL; + pgStatLocalHash =3D NULL; + global_snapshot_is_valid =3D false; +} = -startup_failed: - ereport(LOG, - (errmsg("disabling statistics collector for lack of working socket")))= ; +/* + * pgstat_reset_all() - + * + * Remove the stats file. This is currently used only if WAL recovery is + * needed after a crash. + */ +void +pgstat_reset_all(void) +{ + /* standalone server doesn't use shared stats */ + if (!IsUnderPostmaster) + return; = - if (addrs) - pg_freeaddrinfo_all(hints.ai_family, addrs); + /* we must have shared stats attached */ + Assert (StatsShmem->stats_dsa_handle !=3D DSM_HANDLE_INVALID); = - if (pgStatSock !=3D PGINVALID_SOCKET) - closesocket(pgStatSock); - pgStatSock =3D PGINVALID_SOCKET; + /* Startup must be the only user of shared stats */ + Assert (StatsShmem->refcount =3D=3D 1); = /* - * Adjust GUC variables to suppress useless activity, and for debugging - * purposes (seeing track_counts off is a clue that we failed here). We - * use PGC_S_OVERRIDE because there is no point in trying to turn it bac= k - * on from postgresql.conf without a restart. + * We could directly remove files and recreate the shared memory area. B= ut + * detach then attach for simplicity. */ - SetConfigOption("track_counts", "off", PGC_INTERNAL, PGC_S_OVERRIDE); + pgstat_detach_shared_stats(false); /* Don't write */ + pgstat_attach_shared_stats(); } = -/* - * subroutine for pgstat_reset_all +/* ------------------------------------------------------------ + * Public functions used by backends follow + *------------------------------------------------------------ */ -static void -pgstat_reset_remove_files(const char *directory) + +/* ---------- + * pgstat_report_stat() - + * + * Must be called by processes that performs DML: tcop/postgres.c, logica= l + * receiver processes, SPI worker, etc. to apply the so far collected + * per-table and function usage statistics to the shared statistics hashe= s. + * + * Updates are applied not more frequent than the interval of + * PGSTAT_STAT_MIN_INTERVAL milliseconds. They are also postponed on loc= k + * failure if force is false and there's no pending updates longer than + * PGSTAT_STAT_MAX_INTERVAL milliseconds. Postponed updates are retried = in + * succeeding calls of this function. + * + * Returns the time until the next timing when updates are applied in + * milliseconds if there are no updates holded for more than + * PGSTAT_STAT_MIN_INTERVAL milliseconds. + * + * Note that this is called only out of a transaction, so it is fine to u= se + * transaction stop time as an approximation of current time. + * ---------- + */ +long +pgstat_report_stat(bool force) { - DIR *dir; - struct dirent *entry; - char fname[MAXPGPATH * 2]; + static TimestampTz next_flush =3D 0; + static TimestampTz pending_since =3D 0; + TimestampTz now; + pgstat_flush_stat_context cxt =3D {0}; + bool pending_stats =3D false; + long elapsed; + long secs; + int usecs; + + /* Don't expend a clock check if nothing to do */ + if (area =3D=3D NULL || + ((pgStatTabList =3D=3D NULL || pgStatTabList->tsa_used =3D=3D 0) && + pgStatXactCommit =3D=3D 0 && pgStatXactRollback =3D=3D 0 && + !HAVE_PENDING_DBSTATS() && !have_function_stats)) + return 0; + + now =3D GetCurrentTransactionStopTimestamp(); = - dir =3D AllocateDir(directory); - while ((entry =3D ReadDir(dir, directory)) !=3D NULL) + if (!force) { - int nchars; - Oid tmp_oid; + /* + * Don't flush stats unless it's the time. Returns time to wait in + * milliseconds. + */ + if (now < next_flush) + { + /* Record the oldest pending update if not yet. */ + if (pending_since =3D=3D 0) + pending_since =3D now; + + /* now < next_flush here */ + return (next_flush - now) / 1000; + } = /* - * Skip directory entries that don't match the file names we write. - * See get_dbstat_filename for the database-specific pattern. + * Don't keep pending updates longer than PGSTAT_STAT_MAX_INTERVAL. */ - if (strncmp(entry->d_name, "global.", 7) =3D=3D 0) - nchars =3D 7; - else + if (pending_since > 0) { - nchars =3D 0; - (void) sscanf(entry->d_name, "db_%u.%n", - &tmp_oid, &nchars); - if (nchars <=3D 0) - continue; - /* %u allows leading whitespace, so reject that */ - if (strchr("0123456789", entry->d_name[3]) =3D=3D NULL) - continue; + TimestampDifference(pending_since, now, &secs, &usecs); + elapsed =3D secs * 1000 + usecs /1000; + + if(elapsed > PGSTAT_STAT_MAX_INTERVAL) + force =3D true; } + } = - if (strcmp(entry->d_name + nchars, "tmp") !=3D 0 && - strcmp(entry->d_name + nchars, "stat") !=3D 0) - continue; + /* Flush out table stats */ + if (pgStatTabList !=3D NULL && !pgstat_flush_stat(&cxt, !force)) + pending_stats =3D true; = - snprintf(fname, sizeof(fname), "%s/%s", directory, - entry->d_name); - unlink(fname); + /* Flush out function stats */ + if (pgStatFunctions !=3D NULL && !pgstat_flush_funcstats(&cxt, !force)) + pending_stats =3D true; + + /* Flush out database-wide stats */ + if (HAVE_PENDING_DBSTATS()) + { + if (!pgstat_flush_dbstats(&cxt, !force)) + pending_stats =3D true; } - FreeDir(dir); -} = -/* - * pgstat_reset_all() - - * - * Remove the stats files. This is currently used only if WAL - * recovery is needed after a crash. - */ -void -pgstat_reset_all(void) -{ - pgstat_reset_remove_files(pgstat_stat_directory); - pgstat_reset_remove_files(PGSTAT_STAT_PERMANENT_DIRECTORY); -} + /* Unpin dbentry if pinned */ + if (cxt.mydb_tabhash) + { + dshash_detach(cxt.mydb_tabhash); + unpin_hashes(cxt.mydbentry, cxt.mygeneration); + cxt.mydb_tabhash =3D NULL; + cxt.mydbentry =3D NULL; + } = -#ifdef EXEC_BACKEND + /* Publish the last flush time */ + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + if (shared_globalStats->stats_timestamp < now) + shared_globalStats->stats_timestamp =3D now; + LWLockRelease(StatsLock); = -/* - * pgstat_forkexec() - - * - * Format up the arglist for, then fork and exec, statistics collector pr= ocess - */ -static pid_t -pgstat_forkexec(void) -{ - char *av[10]; - int ac =3D 0; + /* Record how long we are keepnig pending updats. */ + if (pending_stats) + { + /* Preserve the first value */ + if (pending_since =3D=3D 0) + pending_since =3D now; = - av[ac++] =3D "postgres"; - av[ac++] =3D "--forkcol"; - av[ac++] =3D NULL; /* filled in by postmaster_forkexec */ + /* + * It's possible that the retry interval is longer than the limit by + * PGSTAT_STAT_MAX_INTERVAL. We don't bother that since it's not so + * much. + */ + return PGSTAT_STAT_RETRY_INTERVAL; + } = - av[ac] =3D NULL; - Assert(ac < lengthof(av)); + /* Set the next time to update stats */ + next_flush =3D now + PGSTAT_STAT_MIN_INTERVAL * 1000; + pending_since =3D 0; = - return postmaster_forkexec(ac, av); + return 0; } -#endif /* EXEC_BACKEND */ - = /* - * pgstat_start() - + * snapshot_statentry() - Common routine for functions + * pgstat_fetch_stat_*entry() * - * Called from postmaster at startup or after an existing collector - * died. Attempt to fire up a fresh statistics collector. + * Returns the pointer to a snapshot of a shared entry for the key or NU= LL if + * not found. Returned snapshots are stable during the current transacti= on or + * until pgstat_clear_snapshot() is called. * - * Returns PID of child process, or 0 if fail. + * The snapshots are stored in a hash, pointer to which is stored in the + * *HTAB variable pointed by cxt->hash. If not created yet, it is create= d + * using hash_name, hash_entsize in cxt. * - * Note: if fail, we will be called again from the postmaster main loop. + * cxt->dshash points to dshash_table for dbstat entries. If not yet + * attached, it is attached using cxt->dsh_handle. */ -int -pgstat_start(void) +static void * +snapshot_statentry(pgstat_snapshot_param *cxt, Oid key) { - time_t curtime; - pid_t pgStatPid; + PgStat_snapshot *lentry =3D NULL; + size_t keysize =3D cxt->dsh_params->key_size; + size_t dsh_entrysize =3D cxt->dsh_params->entry_size; + bool found; = /* - * Check that the socket is there, else pgstat_init failed and we can do - * nothing useful. + * We don't want so frequent update of stats snapshot. Keep it at least + * for PGSTAT_STAT_MIN_INTERVAL ms. Not postpone but just ignore the cue= . */ - if (pgStatSock =3D=3D PGINVALID_SOCKET) - return 0; + if (clear_snapshot) + { + clear_snapshot =3D false; + + if (pgStatSnapshotContext && + snapshot_globalStats.stats_timestamp < + GetCurrentStatementStartTimestamp() - + PGSTAT_STAT_MIN_INTERVAL * 1000) + { + MemoryContextReset(pgStatSnapshotContext); + + /* Reset variables */ + global_snapshot_is_valid =3D false; + pgStatSnapshotContext =3D NULL; + pgStatLocalHash =3D NULL; + + pgstat_setup_memcxt(); + } + } = /* - * Do nothing if too soon since last collector start. This is a safety - * valve to protect against continuous respawn attempts if the collector - * is dying immediately at launch. Note that since we will be re-called - * from the postmaster main loop, we will get another chance later. + * Create new hash, with rather arbitrary initial number of entries sinc= e + * we don't know how this hash will grow. */ - curtime =3D time(NULL); - if ((unsigned int) (curtime - last_pgstat_start_time) < - (unsigned int) PGSTAT_RESTART_INTERVAL) - return 0; - last_pgstat_start_time =3D curtime; + if (!*cxt->hash) + { + HASHCTL ctl; + + /* + * Create the hash in the stats context + * + * The entry is prepended by common header part represented by + * PgStat_snapshot. + */ + + ctl.keysize =3D keysize; + ctl.entrysize =3D offsetof(PgStat_snapshot, body) + cxt->hash_entsize; + ctl.hcxt =3D pgStatSnapshotContext; + *cxt->hash =3D hash_create(cxt->hash_name, 32, &ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + + lentry =3D hash_search(*cxt->hash, &key, HASH_ENTER, &found); = /* - * Okay, fork off the collector. + * Refer shared hash if not found in the local hash. We return up-to-dat= e + * entries outside a transaction so do the same even if the snapshot is + * found. */ -#ifdef EXEC_BACKEND - switch ((pgStatPid =3D pgstat_forkexec())) -#else - switch ((pgStatPid =3D fork_process())) -#endif + if (!found || !IsTransactionState()) { - case -1: - ereport(LOG, - (errmsg("could not fork statistics collector: %m"))); - return 0; + void *sentry; = -#ifndef EXEC_BACKEND - case 0: - /* in postmaster child ... */ - InitPostmasterChild(); + /* attach shared hash if not given, leave it alone for later use */ + if (!*cxt->dshash) + { + MemoryContext oldcxt; = - /* Close the postmaster's sockets */ - ClosePostmasterPorts(false); + Assert (cxt->dsh_handle !=3D DSM_HANDLE_INVALID); + oldcxt =3D MemoryContextSwitchTo(pgStatSnapshotContext); + *cxt->dshash =3D + dshash_attach(area, cxt->dsh_params, cxt->dsh_handle, NULL); + MemoryContextSwitchTo(oldcxt); + } = - /* Drop our connection to postmaster's shared memory, as well */ - dsm_detach_all(); - PGSharedMemoryDetach(); + sentry =3D dshash_find(*cxt->dshash, &key, false); = - PgstatCollectorMain(0, NULL); - break; -#endif + if (sentry) + { + /* + * In transaction state, it is obvious that we should create local + * cache entries for consistency. If we are not, we return an + * up-to-date entry. Having said that, we need a local copy since + * dshash entry must be released immediately. We share the same + * local hash entry for the purpose. + */ + memcpy(&lentry->body, sentry, dsh_entrysize); + dshash_release_lock(*cxt->dshash, sentry); = - default: - return (int) pgStatPid; + /* then zero out the local additional space if any */ + if (dsh_entrysize < cxt->hash_entsize) + MemSet((char *)&lentry->body + dsh_entrysize, 0, + cxt->hash_entsize - dsh_entrysize); + } + + lentry->negative =3D !sentry; } = - /* shouldn't get here */ - return 0; -} + if (lentry->negative) + return NULL; = -void -allow_immediate_pgstat_restart(void) -{ - last_pgstat_start_time =3D 0; + return &lentry->body; } = -/* ------------------------------------------------------------ - * Public functions used by backends follow - *------------------------------------------------------------ - */ - - -/* ---------- - * pgstat_report_stat() - +/* + * pgstat_flush_stat: Flushes table stats out to shared statistics. * - * Must be called by processes that performs DML: tcop/postgres.c, logica= l - * receiver processes, SPI worker, etc. to send the so far collected - * per-table and function usage statistics to the collector. Note that t= his - * is called only when not within a transaction, so it is fair to use - * transaction stop time as an approximation of current time. - * ---------- + * If nowait is true, returns false if required lock was not acquired + * immediately. In that case, unapplied table stats updates are left alo= ne in + * TabStatusArray to wait for the next chance. cxt holds some dshash rel= ated + * values that we want to carry around while updating shared stats. + * + * Returns true if all stats info are flushed. Caller must detach dshash= es + * stored in cxt after use. */ -void -pgstat_report_stat(bool force) +static bool +pgstat_flush_stat(pgstat_flush_stat_context *cxt, bool nowait) { - /* we assume this inits to all zeroes: */ static const PgStat_TableCounts all_zeroes; - static TimestampTz last_report =3D 0; - - TimestampTz now; - PgStat_MsgTabstat regular_msg; - PgStat_MsgTabstat shared_msg; TabStatusArray *tsa; - int i; + HTAB *new_tsa_hash =3D NULL; + TabStatusArray *dest_tsa =3D pgStatTabList; + int dest_elem =3D 0; + int i; = - /* Don't expend a clock check if nothing to do */ - if ((pgStatTabList =3D=3D NULL || pgStatTabList->tsa_used =3D=3D 0) && - pgStatXactCommit =3D=3D 0 && pgStatXactRollback =3D=3D 0 && - !have_function_stats) - return; - - /* - * Don't send a message unless it's been at least PGSTAT_STAT_INTERVAL - * msec since we last sent one, or the caller wants to force stats out. - */ - now =3D GetCurrentTransactionStopTimestamp(); - if (!force && - !TimestampDifferenceExceeds(last_report, now, PGSTAT_STAT_INTERVAL)) - return; - last_report =3D now; + /* nothing to do, just return */ + if (pgStatTabHash =3D=3D NULL) + return true; = /* * Destroy pgStatTabHash before we start invalidating PgStat_TableEntry - * entries it points to. (Should we fail partway through the loop below= , - * it's okay to have removed the hashtable already --- the only - * consequence is we'd get multiple entries for the same table in the - * pgStatTabList, and that's safe.) + * entries it points to. */ - if (pgStatTabHash) - hash_destroy(pgStatTabHash); + hash_destroy(pgStatTabHash); pgStatTabHash =3D NULL; = /* * Scan through the TabStatusArray struct(s) to find tables that actuall= y - * have counts, and build messages to send. We have to separate shared - * relations from regular ones because the databaseid field in the messa= ge - * header has to depend on that. + * have counts, and try flushing it out to shared stats. We may fail on + * some entries in the array. Leaving the entries being packed at the + * beginning of the array. */ - regular_msg.m_databaseid =3D MyDatabaseId; - shared_msg.m_databaseid =3D InvalidOid; - regular_msg.m_nentries =3D 0; - shared_msg.m_nentries =3D 0; - for (tsa =3D pgStatTabList; tsa !=3D NULL; tsa =3D tsa->tsa_next) { for (i =3D 0; i < tsa->tsa_used; i++) { PgStat_TableStatus *entry =3D &tsa->tsa_entries[i]; - PgStat_MsgTabstat *this_msg; - PgStat_TableEntry *this_ent; = /* Shouldn't have any pending transaction-dependent counts */ Assert(entry->trans =3D=3D NULL); @@ -875,178 +916,352 @@ pgstat_report_stat(bool force) sizeof(PgStat_TableCounts)) =3D=3D 0) continue; = - /* - * OK, insert data into the appropriate message, and send if full. - */ - this_msg =3D entry->t_shared ? &shared_msg : ®ular_msg; - this_ent =3D &this_msg->m_entry[this_msg->m_nentries]; - this_ent->t_id =3D entry->t_id; - memcpy(&this_ent->t_counts, &entry->t_counts, - sizeof(PgStat_TableCounts)); - if (++this_msg->m_nentries >=3D PGSTAT_NUM_TABENTRIES) + /* try to apply the tab stats */ + if (!pgstat_flush_tabstat(cxt, nowait, entry)) { - pgstat_send_tabstat(this_msg); - this_msg->m_nentries =3D 0; + /* + * Failed. Move it to the beginning in TabStatusArray and + * leave it. + */ + TabStatHashEntry *hash_entry; + bool found; + + if (new_tsa_hash =3D=3D NULL) + new_tsa_hash =3D create_tabstat_hash(); + + /* Create hash entry for this entry */ + hash_entry =3D hash_search(new_tsa_hash, &entry->t_id, + HASH_ENTER, &found); + Assert(!found); + + /* + * Move insertion pointer to the next segment if the segment + * is filled up. + */ + if (dest_elem >=3D TABSTAT_QUANTUM) + { + Assert(dest_tsa->tsa_next !=3D NULL); + dest_tsa =3D dest_tsa->tsa_next; + dest_elem =3D 0; + } + + /* + * Pack the entry at the begining of the array. Do nothing if + * no need to be moved. + */ + if (tsa !=3D dest_tsa || i !=3D dest_elem) + { + PgStat_TableStatus *new_entry; + new_entry =3D &dest_tsa->tsa_entries[dest_elem]; + *new_entry =3D *entry; + + /* use new_entry as entry hereafter */ + entry =3D new_entry; + } + + hash_entry->tsa_entry =3D entry; + dest_elem++; } } - /* zero out PgStat_TableStatus structs after use */ - MemSet(tsa->tsa_entries, 0, - tsa->tsa_used * sizeof(PgStat_TableStatus)); - tsa->tsa_used =3D 0; } = + /* zero out unused area of TableStatus */ + dest_tsa->tsa_used =3D dest_elem; + MemSet(&dest_tsa->tsa_entries[dest_elem], 0, + (TABSTAT_QUANTUM - dest_elem) * sizeof(PgStat_TableStatus)); + while (dest_tsa->tsa_next) + { + dest_tsa =3D dest_tsa->tsa_next; + MemSet(dest_tsa->tsa_entries, 0, + dest_tsa->tsa_used * sizeof(PgStat_TableStatus)); + dest_tsa->tsa_used =3D 0; + } + + /* and set the new TabStatusArray hash if any */ + pgStatTabHash =3D new_tsa_hash; + /* - * Send partial messages. Make sure that any pending xact commit/abort - * gets counted, even if there are no table stats to send. + * We no longer need shared database and table entries, but that for my + * database may be used later. */ - if (regular_msg.m_nentries > 0 || - pgStatXactCommit > 0 || pgStatXactRollback > 0) - pgstat_send_tabstat(®ular_msg); - if (shared_msg.m_nentries > 0) - pgstat_send_tabstat(&shared_msg); - - /* Now, send function statistics */ - pgstat_send_funcstats(); + if (cxt->shdb_tabhash) + { + dshash_detach(cxt->shdb_tabhash); + unpin_hashes(cxt->shdbentry, cxt->shgeneration); + cxt->shdb_tabhash =3D NULL; + cxt->shdbentry =3D NULL; + } + + return pgStatTabHash =3D=3D NULL; } = +/* ------- + * Subroutines for pgstat_flush_stat. + * ------- + */ /* - * Subroutine for pgstat_report_stat: finish and send a tabstat message + * pgstat_flush_tabstat: Flushes a table stats entry. + * + * If nowait is true, returns false on lock failure. Dshashes for table= and + * function stats are kept attached in ctx. The caller must detach them = after + * use. + * + * Returns true if the entry is flushed out. */ -static void -pgstat_send_tabstat(PgStat_MsgTabstat *tsmsg) +bool +pgstat_flush_tabstat(pgstat_flush_stat_context *cxt, bool nowait, + PgStat_TableStatus *entry) { - int n; - int len; + Oid dboid =3D entry->t_shared ? InvalidOid : MyDatabaseId; + int table_mode =3D PGSTAT_EXCLUSIVE; + bool updated =3D false; + dshash_table *tabhash; + PgStat_StatDBEntry *dbent; + int generation; + + if (nowait) + table_mode |=3D PGSTAT_NOWAIT; + + /* Attach required table hash if not yet. */ + if ((entry->t_shared ? cxt->shdb_tabhash : cxt->mydb_tabhash) =3D=3D NUL= L) + { + /* + * Return if we don't have corresponding dbentry. It would've been + * removed. + */ + dbent =3D pgstat_get_db_entry(dboid, table_mode, NULL); + if (!dbent) + return false; = - /* It's unlikely we'd get here with no socket, but maybe not impossible = */ - if (pgStatSock =3D=3D PGINVALID_SOCKET) - return; + /* + * We don't hold lock on the dbentry since it cannot be dropped while + * we are working on it. + */ + generation =3D pin_hashes(dbent); + tabhash =3D attach_table_hash(dbent, generation); = - /* - * Report and reset accumulated xact commit/rollback and I/O timings - * whenever we send a normal tabstat message - */ - if (OidIsValid(tsmsg->m_databaseid)) - { - tsmsg->m_xact_commit =3D pgStatXactCommit; - tsmsg->m_xact_rollback =3D pgStatXactRollback; - tsmsg->m_block_read_time =3D pgStatBlockReadTime; - tsmsg->m_block_write_time =3D pgStatBlockWriteTime; - pgStatXactCommit =3D 0; - pgStatXactRollback =3D 0; - pgStatBlockReadTime =3D 0; - pgStatBlockWriteTime =3D 0; + if (entry->t_shared) + { + cxt->shgeneration =3D generation; + cxt->shdbentry =3D dbent; + cxt->shdb_tabhash =3D tabhash; + } + else + { + cxt->mygeneration =3D generation; + cxt->mydbentry =3D dbent; + cxt->mydb_tabhash =3D tabhash; + + /* + * We come here once per database. Take the chance to update + * database-wide stats + */ + LWLockAcquire(&dbent->lock, LW_EXCLUSIVE); + dbent->n_xact_commit +=3D pgStatXactCommit; + dbent->n_xact_rollback +=3D pgStatXactRollback; + dbent->n_block_read_time +=3D pgStatBlockReadTime; + dbent->n_block_write_time +=3D pgStatBlockWriteTime; + LWLockRelease(&dbent->lock); + pgStatXactCommit =3D 0; + pgStatXactRollback =3D 0; + pgStatBlockReadTime =3D 0; + pgStatBlockWriteTime =3D 0; + } + } + else if (entry->t_shared) + { + dbent =3D cxt->shdbentry; + tabhash =3D cxt->shdb_tabhash; } else { - tsmsg->m_xact_commit =3D 0; - tsmsg->m_xact_rollback =3D 0; - tsmsg->m_block_read_time =3D 0; - tsmsg->m_block_write_time =3D 0; + dbent =3D cxt->mydbentry; + tabhash =3D cxt->mydb_tabhash; } = - n =3D tsmsg->m_nentries; - len =3D offsetof(PgStat_MsgTabstat, m_entry[0]) + - n * sizeof(PgStat_TableEntry); = - pgstat_setheader(&tsmsg->m_hdr, PGSTAT_MTYPE_TABSTAT); - pgstat_send(tsmsg, len); + /* + * Local table stats should be applied to both dbentry and tabentry at + * once. Update dbentry only if we could update tabentry. + */ + if (pgstat_update_tabentry(tabhash, entry, nowait)) + { + pgstat_update_dbentry(dbent, entry); + updated =3D true; + } + + return updated; } = /* - * Subroutine for pgstat_report_stat: populate and send a function stat m= essage + * pgstat_flush_funcstats: Flushes function stats. + * + * If nowait is true, returns false on lock failure. Unapplied local has= h + * entryis are left alone. + * + * Returns true if all entries are flushed out. */ -static void -pgstat_send_funcstats(void) +static bool +pgstat_flush_funcstats(pgstat_flush_stat_context *cxt, bool nowait) { /* we assume this inits to all zeroes: */ static const PgStat_FunctionCounts all_zeroes; - - PgStat_MsgFuncstat msg; - PgStat_BackendFunctionEntry *entry; + dshash_table *funchash; HASH_SEQ_STATUS fstat; + PgStat_BackendFunctionEntry *bestat; = + /* nothing to do, just return */ if (pgStatFunctions =3D=3D NULL) - return; + return true; + + /* get dbentry into cxt if not yet */ + if (cxt->mydbentry =3D=3D NULL) + { + int op =3D PGSTAT_EXCLUSIVE; + + if (nowait) + op |=3D PGSTAT_NOWAIT; + + cxt->mydbentry =3D pgstat_get_db_entry(MyDatabaseId, op, NULL); + + if (cxt->mydbentry =3D=3D NULL) + return false; + + cxt->mygeneration =3D pin_hashes(cxt->mydbentry); + } + + funchash =3D attach_function_hash(cxt->mydbentry, cxt->mygeneration); + if (funchash =3D=3D NULL) + return false; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_FUNCSTAT); - msg.m_databaseid =3D MyDatabaseId; - msg.m_nentries =3D 0; + have_function_stats =3D false; = + /* + * Scan through the pgStatFunctions to find functions that actually have + * counts, and try flushing it out to shared stats. + */ hash_seq_init(&fstat, pgStatFunctions); - while ((entry =3D (PgStat_BackendFunctionEntry *) hash_seq_search(&fstat= )) !=3D NULL) + while ((bestat =3D (PgStat_BackendFunctionEntry *) hash_seq_search(&fsta= t)) !=3D NULL) { - PgStat_FunctionEntry *m_ent; + bool found; + PgStat_StatFuncEntry *funcent =3D NULL; = - /* Skip it if no counts accumulated since last time */ - if (memcmp(&entry->f_counts, &all_zeroes, + /* Skip it if no counts accumulated for it so far */ + if (memcmp(&bestat->f_counts, &all_zeroes, sizeof(PgStat_FunctionCounts)) =3D=3D 0) continue; = - /* need to convert format of time accumulators */ - m_ent =3D &msg.m_entry[msg.m_nentries]; - m_ent->f_id =3D entry->f_id; - m_ent->f_numcalls =3D entry->f_counts.f_numcalls; - m_ent->f_total_time =3D INSTR_TIME_GET_MICROSEC(entry->f_counts.f_total= _time); - m_ent->f_self_time =3D INSTR_TIME_GET_MICROSEC(entry->f_counts.f_self_t= ime); + funcent =3D (PgStat_StatFuncEntry *) + dshash_find_or_insert_extended(funchash, (void *) &(bestat->f_id), + &found, nowait); + + /* + * We couldn't acquire lock on the required entry. Leave the local + * entry alone. + */ + if (!funcent) + { + have_function_stats =3D true; + continue; + } = - if (++msg.m_nentries >=3D PGSTAT_NUM_FUNCENTRIES) + /* Initialize if it's new, or add to it. */ + if (!found) + { + funcent->functionid =3D bestat->f_id; + funcent->f_numcalls =3D bestat->f_counts.f_numcalls; + funcent->f_total_time =3D + INSTR_TIME_GET_MICROSEC(bestat->f_counts.f_total_time); + funcent->f_self_time =3D + INSTR_TIME_GET_MICROSEC(bestat->f_counts.f_self_time); + } + else { - pgstat_send(&msg, offsetof(PgStat_MsgFuncstat, m_entry[0]) + - msg.m_nentries * sizeof(PgStat_FunctionEntry)); - msg.m_nentries =3D 0; + funcent->f_numcalls +=3D bestat->f_counts.f_numcalls; + funcent->f_total_time +=3D + INSTR_TIME_GET_MICROSEC(bestat->f_counts.f_total_time); + funcent->f_self_time +=3D + INSTR_TIME_GET_MICROSEC(bestat->f_counts.f_self_time); } + dshash_release_lock(funchash, funcent); = - /* reset the entry's counts */ - MemSet(&entry->f_counts, 0, sizeof(PgStat_FunctionCounts)); + /* reset used counts */ + MemSet(&bestat->f_counts, 0, sizeof(PgStat_FunctionCounts)); } = - if (msg.m_nentries > 0) - pgstat_send(&msg, offsetof(PgStat_MsgFuncstat, m_entry[0]) + - msg.m_nentries * sizeof(PgStat_FunctionEntry)); - - have_function_stats =3D false; + return !have_function_stats; } = +/* + * pgstat_flush_dbstats: Flushes out miscellaneous database stats. + * + * If nowait is true, returns with false on lock failure on dbentry. + * + * Returns true if all stats are flushed out. + */ +static bool +pgstat_flush_dbstats(pgstat_flush_stat_context *cxt, bool nowait) +{ + /* get dbentry if not yet */ + if (cxt->mydbentry =3D=3D NULL) + { + int op =3D PGSTAT_EXCLUSIVE; + if (nowait) + op |=3D PGSTAT_NOWAIT; + + cxt->mydbentry =3D pgstat_get_db_entry(MyDatabaseId, op, NULL); + + /* return if lock failed. */ + if (cxt->mydbentry =3D=3D NULL) + return false; + + /* we use this generation of table /function stats in this turn */ + cxt->mygeneration =3D pin_hashes(cxt->mydbentry); + } + + LWLockAcquire(&cxt->mydbentry->lock, LW_EXCLUSIVE); + if (HAVE_PENDING_CONFLICTS()) + pgstat_flush_recovery_conflict(cxt->mydbentry); + if (BeDBStats.n_deadlocks !=3D 0) + pgstat_flush_deadlock(cxt->mydbentry); + if (BeDBStats.n_tmpfiles !=3D 0) + pgstat_flush_tempfile(cxt->mydbentry); + if (BeDBStats.checksum_failures !=3D NULL) + pgstat_flush_checksum_failure(cxt->mydbentry); + LWLockRelease(&cxt->mydbentry->lock); + + return true; +} = /* ---------- * pgstat_vacuum_stat() - * - * Will tell the collector about objects he can get rid of. + * Remove objects we can get rid of. * ---------- */ void pgstat_vacuum_stat(void) { - HTAB *htab; - PgStat_MsgTabpurge msg; - PgStat_MsgFuncpurge f_msg; - HASH_SEQ_STATUS hstat; + HTAB *oidtab; + dshash_seq_status dshstat; PgStat_StatDBEntry *dbentry; - PgStat_StatTabEntry *tabentry; - PgStat_StatFuncEntry *funcentry; - int len; = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + /* we don't collect stats under standalone mode */ + if (!IsUnderPostmaster) return; = /* - * If not done for this transaction, read the statistics collector stats - * file into some hash tables. - */ - backend_read_statsfile(); - - /* * Read pg_database and make a list of OIDs of all existing databases */ - htab =3D pgstat_collect_oids(DatabaseRelationId, Anum_pg_database_oid); + oidtab =3D pgstat_collect_oids(DatabaseRelationId, Anum_pg_database_oid)= ; = /* - * Search the database hash table for dead databases and tell the - * collector to drop them. + * Search the database hash table for dead databases and drop them + * from the hash. */ - hash_seq_init(&hstat, pgStatDBHash); - while ((dbentry =3D (PgStat_StatDBEntry *) hash_seq_search(&hstat)) !=3D= NULL) + + dshash_seq_init(&dshstat, pgStatDBHash, false, true); + while ((dbentry =3D (PgStat_StatDBEntry *) dshash_seq_next(&dshstat)) !=3D= NULL) { Oid dbid =3D dbentry->databaseid; = @@ -1054,137 +1269,43 @@ pgstat_vacuum_stat(void) = /* the DB entry for shared tables (with InvalidOid) is never dropped */ if (OidIsValid(dbid) && - hash_search(htab, (void *) &dbid, HASH_FIND, NULL) =3D=3D NULL) + hash_search(oidtab, (void *) &dbid, HASH_FIND, NULL) =3D=3D NULL) pgstat_drop_database(dbid); } = /* Clean up */ - hash_destroy(htab); + hash_destroy(oidtab); = /* * Lookup our own database entry; if not found, nothing more to do. */ - dbentry =3D (PgStat_StatDBEntry *) hash_search(pgStatDBHash, - (void *) &MyDatabaseId, - HASH_FIND, NULL); - if (dbentry =3D=3D NULL || dbentry->tables =3D=3D NULL) + dbentry =3D pgstat_get_db_entry(MyDatabaseId, PGSTAT_EXCLUSIVE, NULL); + if (!dbentry) return; = /* * Similarly to above, make a list of all known relations in this DB. */ - htab =3D pgstat_collect_oids(RelationRelationId, Anum_pg_class_oid); - - /* - * Initialize our messages table counter to zero - */ - msg.m_nentries =3D 0; + oidtab =3D pgstat_collect_oids(RelationRelationId, Anum_pg_class_oid); = /* * Check for all tables listed in stats hashtable if they still exist. + * Stats cache is useless here so directly search the shared hash. */ - hash_seq_init(&hstat, dbentry->tables); - while ((tabentry =3D (PgStat_StatTabEntry *) hash_seq_search(&hstat)) !=3D= NULL) - { - Oid tabid =3D tabentry->tableid; - - CHECK_FOR_INTERRUPTS(); - - if (hash_search(htab, (void *) &tabid, HASH_FIND, NULL) !=3D NULL) - continue; - - /* - * Not there, so add this table's Oid to the message - */ - msg.m_tableid[msg.m_nentries++] =3D tabid; - - /* - * If the message is full, send it out and reinitialize to empty - */ - if (msg.m_nentries >=3D PGSTAT_NUM_TABPURGE) - { - len =3D offsetof(PgStat_MsgTabpurge, m_tableid[0]) - + msg.m_nentries * sizeof(Oid); - - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_TABPURGE); - msg.m_databaseid =3D MyDatabaseId; - pgstat_send(&msg, len); - - msg.m_nentries =3D 0; - } - } - - /* - * Send the rest - */ - if (msg.m_nentries > 0) - { - len =3D offsetof(PgStat_MsgTabpurge, m_tableid[0]) - + msg.m_nentries * sizeof(Oid); - - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_TABPURGE); - msg.m_databaseid =3D MyDatabaseId; - pgstat_send(&msg, len); - } - - /* Clean up */ - hash_destroy(htab); + pgstat_remove_useless_entries(dbentry->tables, &dsh_tblparams, oidtab); = /* - * Now repeat the above steps for functions. However, we needn't bother - * in the common case where no function stats are being collected. + * Repeat the above but we needn't bother in the common case where no + * function stats are being collected. */ - if (dbentry->functions !=3D NULL && - hash_get_num_entries(dbentry->functions) > 0) + if (dbentry->functions !=3D DSM_HANDLE_INVALID) { - htab =3D pgstat_collect_oids(ProcedureRelationId, Anum_pg_proc_oid); - - pgstat_setheader(&f_msg.m_hdr, PGSTAT_MTYPE_FUNCPURGE); - f_msg.m_databaseid =3D MyDatabaseId; - f_msg.m_nentries =3D 0; - - hash_seq_init(&hstat, dbentry->functions); - while ((funcentry =3D (PgStat_StatFuncEntry *) hash_seq_search(&hstat))= !=3D NULL) - { - Oid funcid =3D funcentry->functionid; - - CHECK_FOR_INTERRUPTS(); - - if (hash_search(htab, (void *) &funcid, HASH_FIND, NULL) !=3D NULL) - continue; - - /* - * Not there, so add this function's Oid to the message - */ - f_msg.m_functionid[f_msg.m_nentries++] =3D funcid; - - /* - * If the message is full, send it out and reinitialize to empty - */ - if (f_msg.m_nentries >=3D PGSTAT_NUM_FUNCPURGE) - { - len =3D offsetof(PgStat_MsgFuncpurge, m_functionid[0]) - + f_msg.m_nentries * sizeof(Oid); - - pgstat_send(&f_msg, len); - - f_msg.m_nentries =3D 0; - } - } - - /* - * Send the rest - */ - if (f_msg.m_nentries > 0) - { - len =3D offsetof(PgStat_MsgFuncpurge, m_functionid[0]) - + f_msg.m_nentries * sizeof(Oid); - - pgstat_send(&f_msg, len); - } + oidtab =3D pgstat_collect_oids(ProcedureRelationId, Anum_pg_proc_oid); = - hash_destroy(htab); + pgstat_remove_useless_entries(dbentry->functions, &dsh_funcparams, + oidtab); } + dshash_release_lock(pgStatDBHash, dbentry); } = = @@ -1238,66 +1359,99 @@ pgstat_collect_oids(Oid catalogid, AttrNumber anum= _oid) return htab; } = - -/* ---------- - * pgstat_drop_database() - +/* + * pgstat_remove_useless_entries - Remove useless entries from per + * table/function dshashes. * - * Tell the collector that we just dropped a database. - * (If the message gets lost, we will still clean the dead DB eventually - * via future invocations of pgstat_vacuum_stat().) - * ---------- + * Scan the dshash specified by dshhandle removing entries that are not = in + * oidtab. oidtab is destroyed before returning. */ void -pgstat_drop_database(Oid databaseid) +pgstat_remove_useless_entries(const dshash_table_handle dshhandle, + const dshash_parameters *dshparams, + HTAB *oidtab) { - PgStat_MsgDropdb msg; + dshash_table *dshtable; + dshash_seq_status dshstat; + void *ent; = - if (pgStatSock =3D=3D PGINVALID_SOCKET) - return; + dshtable =3D dshash_attach(area, dshparams, dshhandle, 0); + dshash_seq_init(&dshstat, dshtable, false, true); = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_DROPDB); - msg.m_databaseid =3D databaseid; - pgstat_send(&msg, sizeof(msg)); -} + while ((ent =3D dshash_seq_next(&dshstat)) !=3D NULL) + { + CHECK_FOR_INTERRUPTS(); + + /* The first member of the entries must be Oid */ + if (hash_search(oidtab, ent, HASH_FIND, NULL) !=3D NULL) + continue; = + /* Not there, so purge this entry */ + dshash_delete_entry(dshtable, ent); + } + dshash_detach(dshtable); + hash_destroy(oidtab); +} = /* ---------- - * pgstat_drop_relation() - + * pgstat_drop_database() - * - * Tell the collector that we just dropped a relation. - * (If the message gets lost, we will still clean the dead entry eventual= ly - * via future invocations of pgstat_vacuum_stat().) + * Remove entry for the database that we just dropped. * - * Currently not used for lack of any good place to call it; we rely - * entirely on pgstat_vacuum_stat() to clean out stats for dead rels. + * If some stats are flushed after this, this entry will be re-created bu= t we + * will still clean the dead DB eventually via future invocations of + * pgstat_vacuum_stat(). * ---------- */ -#ifdef NOT_USED void -pgstat_drop_relation(Oid relid) +pgstat_drop_database(Oid databaseid) { - PgStat_MsgTabpurge msg; - int len; + PgStat_StatDBEntry *dbentry; + + Assert (OidIsValid(databaseid)); = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + if (!IsUnderPostmaster || !pgStatDBHash) return; = - msg.m_tableid[0] =3D relid; - msg.m_nentries =3D 1; + /* + * Lookup the database in the hashtable with exclusive lock. + */ + dbentry =3D pgstat_get_db_entry(databaseid, PGSTAT_EXCLUSIVE, NULL); + + /* + * If found, remove it. + */ + if (dbentry) + { + /* LWLock is needed to rewrite */ + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); = - len =3D offsetof(PgStat_MsgTabpurge, m_tableid[0]) + sizeof(Oid); + /* No one is using tables/functions in this dbentry */ + Assert(dbentry->refcnt =3D=3D 0); = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_TABPURGE); - msg.m_databaseid =3D MyDatabaseId; - pgstat_send(&msg, len); -} -#endif /* NOT_USED */ + /* Remove table/function stats dshash first. */ + if (dbentry->tables !=3D DSM_HANDLE_INVALID) + { + dshash_table *tbl =3D + dshash_attach(area, &dsh_tblparams, dbentry->tables, 0); + dshash_destroy(tbl); + } + if (dbentry->functions !=3D DSM_HANDLE_INVALID) + { + dshash_table *tbl =3D + dshash_attach(area, &dsh_funcparams, dbentry->functions, 0); + dshash_destroy(tbl); + } + LWLockRelease(&dbentry->lock); = + dshash_delete_entry(pgStatDBHash, (void *)dbentry); + } +} = /* ---------- * pgstat_reset_counters() - * - * Tell the statistics collector to reset counters for our database. + * Reset counters for our database. * * Permission checking for this function is managed through the normal * GRANT system. @@ -1306,20 +1460,32 @@ pgstat_drop_relation(Oid relid) void pgstat_reset_counters(void) { - PgStat_MsgResetcounter msg; + PgStat_StatDBEntry *dbentry; + PgStat_TableLookupResult status; + + if (!pgStatDBHash) + return; = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + /* + * Lookup the database in the hashtable. Nothing to do if not there. + */ + dbentry =3D pgstat_get_db_entry(MyDatabaseId, PGSTAT_EXCLUSIVE, &status)= ; + + if (!dbentry) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETCOUNTER); - msg.m_databaseid =3D MyDatabaseId; - pgstat_send(&msg, sizeof(msg)); + /* This database is active, safe to release the lock immediately. */ + dshash_release_lock(pgStatDBHash, dbentry); + + /* Reset database-level stats. */ + reset_dbentry_counters(dbentry); + } = /* ---------- * pgstat_reset_shared_counters() - * - * Tell the statistics collector to reset cluster-wide shared counters. + * Reset cluster-wide shared counters. * * Permission checking for this function is managed through the normal * GRANT system. @@ -1328,29 +1494,37 @@ pgstat_reset_counters(void) void pgstat_reset_shared_counters(const char *target) { - PgStat_MsgResetsharedcounter msg; - - if (pgStatSock =3D=3D PGINVALID_SOCKET) - return; - + /* Reset the archiver statistics for the cluster. */ if (strcmp(target, "archiver") =3D=3D 0) - msg.m_resettarget =3D RESET_ARCHIVER; + { + TimestampTz now =3D GetCurrentTimestamp(); + + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + MemSet(shared_archiverStats, 0, sizeof(*shared_archiverStats)); + shared_archiverStats->stat_reset_timestamp =3D now; + LWLockRelease(StatsLock); + } + /* Reset the bgwriter statistics for the cluster. */ else if (strcmp(target, "bgwriter") =3D=3D 0) - msg.m_resettarget =3D RESET_BGWRITER; + { + TimestampTz now =3D GetCurrentTimestamp(); + + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + MemSet(shared_globalStats, 0, sizeof(*shared_globalStats)); + shared_globalStats->stat_reset_timestamp =3D now; + LWLockRelease(StatsLock); + } else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized reset target: \"%s\"", target), errhint("Target must be \"archiver\" or \"bgwriter\"."))); - - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETSHAREDCOUNTER); - pgstat_send(&msg, sizeof(msg)); } = /* ---------- * pgstat_reset_single_counter() - * - * Tell the statistics collector to reset a single counter. + * Reset a single counter. * * Permission checking for this function is managed through the normal * GRANT system. @@ -1359,20 +1533,45 @@ pgstat_reset_shared_counters(const char *target) void pgstat_reset_single_counter(Oid objoid, PgStat_Single_Reset_Type type) { - PgStat_MsgResetsinglecounter msg; + PgStat_StatDBEntry *dbentry; + TimestampTz ts; + int generation; + + dbentry =3D pgstat_get_db_entry(MyDatabaseId, PGSTAT_EXCLUSIVE, NULL); = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + if (!dbentry) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETSINGLECOUNTER); - msg.m_databaseid =3D MyDatabaseId; - msg.m_resettype =3D type; - msg.m_objectid =3D objoid; + /* This database is active, safe to release the lock immediately. */ + generation =3D pin_hashes(dbentry); = - pgstat_send(&msg, sizeof(msg)); -} + /* Set the reset timestamp for the whole database */ + ts =3D GetCurrentTimestamp(); + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); + dbentry->stat_reset_timestamp =3D ts; + LWLockRelease(&dbentry->lock); = -/* ---------- + /* Remove object if it exists, ignore if not */ + if (type =3D=3D RESET_TABLE) + { + dshash_table *t =3D attach_table_hash(dbentry, generation); + dshash_delete_key(t, (void *) &objoid); + dshash_detach(t); + } + + if (type =3D=3D RESET_FUNCTION) + { + dshash_table *t =3D attach_function_hash(dbentry, generation); + if (t) + { + dshash_delete_key(t, (void *) &objoid); + dshash_detach(t); + } + } + unpin_hashes(dbentry, generation); +} + +/* ---------- * pgstat_report_autovac() - * * Called from autovacuum.c to report startup of an autovacuum process. @@ -1383,48 +1582,81 @@ pgstat_reset_single_counter(Oid objoid, PgStat_Sin= gle_Reset_Type type) void pgstat_report_autovac(Oid dboid) { - PgStat_MsgAutovacStart msg; + PgStat_StatDBEntry *dbentry; + TimestampTz ts; = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + /* return if we are not collecting stats */ + if (!area) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_AUTOVAC_START); - msg.m_databaseid =3D dboid; - msg.m_start_time =3D GetCurrentTimestamp(); + /* + * Store the last autovacuum time in the database's hashtable entry. + */ + dbentry =3D pgstat_get_db_entry(dboid, PGSTAT_EXCLUSIVE, NULL); + dshash_release_lock(pgStatDBHash, dbentry); + + ts =3D GetCurrentTimestamp(); = - pgstat_send(&msg, sizeof(msg)); + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); + dbentry->last_autovac_time =3D ts; + LWLockRelease(&dbentry->lock); } = = /* --------- * pgstat_report_vacuum() - * - * Tell the collector about the table we just vacuumed. + * Report about the table we just vacuumed. * --------- */ void pgstat_report_vacuum(Oid tableoid, bool shared, PgStat_Counter livetuples, PgStat_Counter deadtuples) { - PgStat_MsgVacuum msg; + Oid dboid; + PgStat_StatDBEntry *dbentry; + PgStat_StatTabEntry *tabentry; + dshash_table *table; + int generation; = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + /* return if we are not collecting stats */ + if (!area) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_VACUUM); - msg.m_databaseid =3D shared ? InvalidOid : MyDatabaseId; - msg.m_tableoid =3D tableoid; - msg.m_autovacuum =3D IsAutoVacuumWorkerProcess(); - msg.m_vacuumtime =3D GetCurrentTimestamp(); - msg.m_live_tuples =3D livetuples; - msg.m_dead_tuples =3D deadtuples; - pgstat_send(&msg, sizeof(msg)); + dboid =3D shared ? InvalidOid : MyDatabaseId; + + /* + * Store the data in the table's hash table entry. + */ + dbentry =3D pgstat_get_db_entry(dboid, PGSTAT_EXCLUSIVE, NULL); + generation =3D pin_hashes(dbentry); + table =3D attach_table_hash(dbentry, generation); + + tabentry =3D pgstat_get_tab_entry(table, tableoid, true); + + tabentry->n_live_tuples =3D livetuples; + tabentry->n_dead_tuples =3D deadtuples; + + if (IsAutoVacuumWorkerProcess()) + { + tabentry->autovac_vacuum_timestamp =3D GetCurrentTimestamp(); + tabentry->autovac_vacuum_count++; + } + else + { + tabentry->vacuum_timestamp =3D GetCurrentTimestamp(); + tabentry->vacuum_count++; + } + dshash_release_lock(table, tabentry); + + dshash_detach(table); + unpin_hashes(dbentry, generation); } = /* -------- * pgstat_report_analyze() - * - * Tell the collector about the table we just analyzed. + * Report about the table we just analyzed. * * Caller must provide new live- and dead-tuples estimates, as well as a * flag indicating whether to reset the changes_since_analyze counter. @@ -1435,9 +1667,14 @@ pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter) { - PgStat_MsgAnalyze msg; + Oid dboid; + PgStat_StatDBEntry *dbentry; + PgStat_StatTabEntry *tabentry; + dshash_table *table; + int generation; = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + /* return if we are not collecting stats */ + if (!area) return; = /* @@ -1466,78 +1703,153 @@ pgstat_report_analyze(Relation rel, deadtuples =3D Max(deadtuples, 0); } = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_ANALYZE); - msg.m_databaseid =3D rel->rd_rel->relisshared ? InvalidOid : MyDatabaseI= d; - msg.m_tableoid =3D RelationGetRelid(rel); - msg.m_autovacuum =3D IsAutoVacuumWorkerProcess(); - msg.m_resetcounter =3D resetcounter; - msg.m_analyzetime =3D GetCurrentTimestamp(); - msg.m_live_tuples =3D livetuples; - msg.m_dead_tuples =3D deadtuples; - pgstat_send(&msg, sizeof(msg)); + dboid =3D rel->rd_rel->relisshared ? InvalidOid : MyDatabaseId; + + /* + * Store the data in the table's hashtable entry. + */ + dbentry =3D pgstat_get_db_entry(dboid, PGSTAT_EXCLUSIVE, NULL); + generation =3D pin_hashes(dbentry); + table =3D attach_table_hash(dbentry, generation); + tabentry =3D pgstat_get_tab_entry(table, RelationGetRelid(rel), true); + + tabentry->n_live_tuples =3D livetuples; + tabentry->n_dead_tuples =3D deadtuples; + + /* + * If commanded, reset changes_since_analyze to zero. This forgets any + * changes that were committed while the ANALYZE was in progress, but we + * have no good way to estimate how many of those there were. + */ + if (resetcounter) + tabentry->changes_since_analyze =3D 0; + + if (IsAutoVacuumWorkerProcess()) + { + tabentry->autovac_analyze_timestamp =3D GetCurrentTimestamp(); + tabentry->autovac_analyze_count++; + } + else + { + tabentry->analyze_timestamp =3D GetCurrentTimestamp(); + tabentry->analyze_count++; + } + dshash_release_lock(table, tabentry); + + dshash_detach(table); + unpin_hashes(dbentry, generation); } = /* -------- * pgstat_report_recovery_conflict() - * - * Tell the collector about a Hot Standby recovery conflict. + * Report a Hot Standby recovery conflict. * -------- */ void pgstat_report_recovery_conflict(int reason) { - PgStat_MsgRecoveryConflict msg; + PgStat_StatDBEntry *dbentry; + PgStat_TableLookupResult status; + + /* return if we are not collecting stats */ + if (!area) + return; + + switch (reason) + { + case PROCSIG_RECOVERY_CONFLICT_DATABASE: + + /* + * Since we drop the information about the database as soon as it + * replicates, there is no point in counting these conflicts. + */ + break; + case PROCSIG_RECOVERY_CONFLICT_TABLESPACE: + BeDBStats.n_conflict_tablespace++; + break; + case PROCSIG_RECOVERY_CONFLICT_LOCK: + BeDBStats.n_conflict_lock++; + break; + case PROCSIG_RECOVERY_CONFLICT_SNAPSHOT: + BeDBStats.n_conflict_snapshot++; + break; + case PROCSIG_RECOVERY_CONFLICT_BUFFERPIN: + BeDBStats.n_conflict_bufferpin++; + break; + case PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK: + BeDBStats.n_conflict_startup_deadlock++; + break; + } + + dbentry =3D pgstat_get_db_entry(MyDatabaseId, + PGSTAT_EXCLUSIVE | PGSTAT_NOWAIT, + &status); = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + if (status =3D=3D LOCK_FAILED) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RECOVERYCONFLICT); - msg.m_databaseid =3D MyDatabaseId; - msg.m_reason =3D reason; - pgstat_send(&msg, sizeof(msg)); + /* We had a chance to flush immediately */ + pgstat_flush_recovery_conflict(dbentry); + + dshash_release_lock(pgStatDBHash, dbentry); +} + +/* + * flush recovery conflict stats + */ +static void +pgstat_flush_recovery_conflict(PgStat_StatDBEntry *dbentry) +{ + dbentry->n_conflict_tablespace +=3D BeDBStats.n_conflict_tablespace; + dbentry->n_conflict_lock +=3D BeDBStats.n_conflict_lock; + dbentry->n_conflict_snapshot +=3D BeDBStats.n_conflict_snapshot; + dbentry->n_conflict_bufferpin +=3D BeDBStats.n_conflict_bufferpin; + dbentry->n_conflict_startup_deadlock +=3D BeDBStats.n_conflict_startup_d= eadlock; + + BeDBStats.n_conflict_tablespace =3D 0; + BeDBStats.n_conflict_lock =3D 0; + BeDBStats.n_conflict_snapshot =3D 0; + BeDBStats.n_conflict_bufferpin =3D 0; + BeDBStats.n_conflict_startup_deadlock =3D 0; } = /* -------- * pgstat_report_deadlock() - * - * Tell the collector about a deadlock detected. + * Report a deadlock detected. * -------- */ void pgstat_report_deadlock(void) { - PgStat_MsgDeadlock msg; + PgStat_StatDBEntry *dbentry; + PgStat_TableLookupResult status; = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + /* return if we are not collecting stats */ + if (!area) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_DEADLOCK); - msg.m_databaseid =3D MyDatabaseId; - pgstat_send(&msg, sizeof(msg)); -} - + BeDBStats.n_deadlocks++; = + dbentry =3D pgstat_get_db_entry(MyDatabaseId, + PGSTAT_EXCLUSIVE | PGSTAT_NOWAIT, + &status); = -/* -------- - * pgstat_report_checksum_failures_in_db() - - * - * Tell the collector about one or more checksum failures. - * -------- - */ -void -pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount) -{ - PgStat_MsgChecksumFailure msg; - - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + if (status =3D=3D LOCK_FAILED) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_CHECKSUMFAILURE); - msg.m_databaseid =3D dboid; - msg.m_failurecount =3D failurecount; - msg.m_failure_time =3D GetCurrentTimestamp(); + dshash_release_lock(pgStatDBHash, dbentry); +} = - pgstat_send(&msg, sizeof(msg)); +/* + * flush dead lock stats + */ +static void +pgstat_flush_deadlock(PgStat_StatDBEntry *dbentry) +{ + dbentry->n_deadlocks +=3D BeDBStats.n_deadlocks; + BeDBStats.n_deadlocks =3D 0; } = /* -------- @@ -1555,60 +1867,153 @@ pgstat_report_checksum_failure(void) /* -------- * pgstat_report_tempfile() - * - * Tell the collector about a temporary file. + * Report a temporary file. * -------- */ void pgstat_report_tempfile(size_t filesize) { - PgStat_MsgTempFile msg; + PgStat_StatDBEntry *dbentry; + PgStat_TableLookupResult status; + + /* return if we are not collecting stats */ + if (!area) + return; + + if (filesize > 0) /* Is there a case where filesize is really 0? */ + { + BeDBStats.tmpfilesize +=3D filesize; /* needs check overflow */ + BeDBStats.n_tmpfiles++; + } = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + if (BeDBStats.n_tmpfiles =3D=3D 0) return; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_TEMPFILE); - msg.m_databaseid =3D MyDatabaseId; - msg.m_filesize =3D filesize; - pgstat_send(&msg, sizeof(msg)); + dbentry =3D pgstat_get_db_entry(MyDatabaseId, + PGSTAT_EXCLUSIVE | PGSTAT_NOWAIT, + &status); + + if (status =3D=3D LOCK_FAILED) + return; + + /* We had a chance to flush immediately */ + pgstat_flush_tempfile(dbentry); + + dshash_release_lock(pgStatDBHash, dbentry); } = +/* + * flush temporary file stats + */ +static void +pgstat_flush_tempfile(PgStat_StatDBEntry *dbentry) +{ = -/* ---------- - * pgstat_ping() - + dbentry->n_temp_bytes +=3D BeDBStats.tmpfilesize; + dbentry->n_temp_files +=3D BeDBStats.n_tmpfiles; + BeDBStats.tmpfilesize =3D 0; + BeDBStats.n_tmpfiles =3D 0; +} + +/* -------- + * pgstat_report_checksum_failures_in_db(dboid, failure_count) - * - * Send some junk data to the collector to increase traffic. - * ---------- + * Tell the collector about one or more checksum failures. + * -------- */ void -pgstat_ping(void) +pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount) { - PgStat_MsgDummy msg; + PgStat_StatDBEntry *dbentry; + PgStat_TableLookupResult status; + ChecksumFailureEnt *failent =3D NULL; + + /* return if we are not collecting stats */ + if (!area) + return; + + if (BeDBStats.checksum_failures !=3D NULL) + { + failent =3D hash_search(BeDBStats.checksum_failures, &dboid, + HASH_FIND, NULL); + if (failent) + failurecount +=3D failent->count; + } + + if (failurecount =3D=3D 0) + return; + + dbentry =3D pgstat_get_db_entry(MyDatabaseId, + PGSTAT_EXCLUSIVE | PGSTAT_NOWAIT, + &status); + + if (status =3D=3D LOCK_FAILED) + { + if (!failent) + { + if (!BeDBStats.checksum_failures) + { + HASHCTL ctl; = - if (pgStatSock =3D=3D PGINVALID_SOCKET) + ctl.keysize =3D sizeof(Oid); + ctl.entrysize =3D sizeof(ChecksumFailureEnt); + BeDBStats.checksum_failures =3D + hash_create("pgstat checksum failure count hash", + 32, &ctl, HASH_ELEM | HASH_BLOBS); + } + + failent =3D hash_search(BeDBStats.checksum_failures, + &dboid, HASH_ENTER, NULL); + } + + failent->count =3D failurecount; return; + } + + /* We have a chance to flush immediately */ + dbentry->n_checksum_failures +=3D failurecount; + BeDBStats.checksum_failures =3D NULL; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_DUMMY); - pgstat_send(&msg, sizeof(msg)); + dshash_release_lock(pgStatDBHash, dbentry); } = -/* ---------- - * pgstat_send_inquiry() - - * - * Notify collector that we need fresh data. - * ---------- +/* + * flush checkpoint failure count for all databases */ static void -pgstat_send_inquiry(TimestampTz clock_time, TimestampTz cutoff_time, Oid = databaseid) +pgstat_flush_checksum_failure(PgStat_StatDBEntry *dbentry) { - PgStat_MsgInquiry msg; + HASH_SEQ_STATUS stat; + ChecksumFailureEnt *ent; + bool release_dbent; = - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_INQUIRY); - msg.clock_time =3D clock_time; - msg.cutoff_time =3D cutoff_time; - msg.databaseid =3D databaseid; - pgstat_send(&msg, sizeof(msg)); -} + if (BeDBStats.checksum_failures =3D=3D NULL) + return; + + hash_seq_init(&stat, BeDBStats.checksum_failures); + while ((ent =3D (ChecksumFailureEnt *) hash_seq_search(&stat)) !=3D NULL= ) + { + release_dbent =3D false; + + if (dbentry->databaseid !=3D ent->dboid) + { + dbentry =3D pgstat_get_db_entry(ent->dboid, + PGSTAT_EXCLUSIVE, NULL); + if (!dbentry) + continue; + + release_dbent =3D true; + } + + dbentry->n_checksum_failures +=3D ent->count; = + if (release_dbent) + dshash_release_lock(pgStatDBHash, dbentry); + } + + hash_destroy(BeDBStats.checksum_failures); + BeDBStats.checksum_failures =3D NULL; +} = /* * Initialize function call usage data. @@ -1760,7 +2165,8 @@ pgstat_initstats(Relation rel) return; } = - if (pgStatSock =3D=3D PGINVALID_SOCKET || !pgstat_track_counts) + /* return if we are not collecting stats */ + if (!area) { /* We're not counting at all */ rel->pgstat_info =3D NULL; @@ -1780,6 +2186,24 @@ pgstat_initstats(Relation rel) } = /* + * create_tabstat_hash - create local hash as transactional storage + */ +static HTAB * +create_tabstat_hash(void) +{ + HASHCTL ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize =3D sizeof(Oid); + ctl.entrysize =3D sizeof(TabStatHashEntry); + + return hash_create("pgstat TabStatusArray lookup hash table", + TABSTAT_QUANTUM, + &ctl, + HASH_ELEM | HASH_BLOBS); +} + +/* * get_tabstat_entry - find or create a PgStat_TableStatus entry for rel */ static PgStat_TableStatus * @@ -1794,18 +2218,7 @@ get_tabstat_entry(Oid rel_id, bool isshared) * Create hash table if we don't have it already. */ if (pgStatTabHash =3D=3D NULL) - { - HASHCTL ctl; - - memset(&ctl, 0, sizeof(ctl)); - ctl.keysize =3D sizeof(Oid); - ctl.entrysize =3D sizeof(TabStatHashEntry); - - pgStatTabHash =3D hash_create("pgstat TabStatusArray lookup hash table"= , - TABSTAT_QUANTUM, - &ctl, - HASH_ELEM | HASH_BLOBS); - } + pgStatTabHash =3D create_tabstat_hash(); = /* * Find an entry or create a new one. @@ -2418,30 +2831,33 @@ pgstat_twophase_postabort(TransactionId xid, uint1= 6 info, /* ---------- * pgstat_fetch_stat_dbentry() - * - * Support function for the SQL-callable pgstat* functions. Returns - * the collected statistics for one database or NULL. NULL doesn't mean - * that the database doesn't exist, it is just not yet known by the - * collector, so the caller is better off to report ZERO instead. - * ---------- + * Find database stats entry on backends. The returned entries are cached + * until transaction end or pgstat_clear_snapshot() is called. */ PgStat_StatDBEntry * pgstat_fetch_stat_dbentry(Oid dbid) { - /* - * If not done for this transaction, read the statistics collector stats - * file into some hash tables. - */ - backend_read_statsfile(); - - /* - * Lookup the requested database; return NULL if not found - */ - return (PgStat_StatDBEntry *) hash_search(pgStatDBHash, - (void *) &dbid, - HASH_FIND, NULL); + /* context for snapshot_statentry */ + static pgstat_snapshot_param param =3D + { + .hash_name =3D "local database stats hash", + .hash_entsize =3D sizeof(PgStat_StatDBEntry), + .dsh_handle =3D DSM_HANDLE_INVALID, /* already attached */ + .dsh_params =3D &dsh_dbparams, + .hash =3D &pgStatLocalHash, + .dshash =3D &pgStatDBHash + }; + + /* should be called from backends */ + Assert(IsUnderPostmaster); + + /* If not done for this transaction, take a snapshot of global stats */ + pgstat_snapshot_global_stats(); + + /* caller doesn't have a business with snapshot-local members */ + return (PgStat_StatDBEntry *) snapshot_statentry(¶m, dbid); } = - /* ---------- * pgstat_fetch_stat_tabentry() - * @@ -2454,51 +2870,66 @@ pgstat_fetch_stat_dbentry(Oid dbid) PgStat_StatTabEntry * pgstat_fetch_stat_tabentry(Oid relid) { - Oid dbid; PgStat_StatDBEntry *dbentry; PgStat_StatTabEntry *tabentry; = - /* - * If not done for this transaction, read the statistics collector stats - * file into some hash tables. - */ - backend_read_statsfile(); + /* Lookup our database, then look in its table hash table. */ + dbentry =3D pgstat_fetch_stat_dbentry(MyDatabaseId); + if (dbentry =3D=3D NULL) + return NULL; = - /* - * Lookup our database, then look in its table hash table. - */ - dbid =3D MyDatabaseId; - dbentry =3D (PgStat_StatDBEntry *) hash_search(pgStatDBHash, - (void *) &dbid, - HASH_FIND, NULL); - if (dbentry !=3D NULL && dbentry->tables !=3D NULL) - { - tabentry =3D (PgStat_StatTabEntry *) hash_search(dbentry->tables, - (void *) &relid, - HASH_FIND, NULL); - if (tabentry) - return tabentry; - } + tabentry =3D pgstat_fetch_stat_tabentry_extended(dbentry, relid); + if (tabentry !=3D NULL) + return tabentry; = /* * If we didn't find it, maybe it's a shared table. */ - dbid =3D InvalidOid; - dbentry =3D (PgStat_StatDBEntry *) hash_search(pgStatDBHash, - (void *) &dbid, - HASH_FIND, NULL); - if (dbentry !=3D NULL && dbentry->tables !=3D NULL) - { - tabentry =3D (PgStat_StatTabEntry *) hash_search(dbentry->tables, - (void *) &relid, - HASH_FIND, NULL); - if (tabentry) - return tabentry; - } + dbentry =3D pgstat_fetch_stat_dbentry(InvalidOid); + if (dbentry =3D=3D NULL) + return NULL; + + tabentry =3D pgstat_fetch_stat_tabentry_extended(dbentry, relid); + if (tabentry !=3D NULL) + return tabentry; = return NULL; } = +/* ---------- + * pgstat_fetch_stat_tabentry_extended() - + * + * Find table stats entry on backends. The returned entries are cached un= til + * transaction end or pgstat_clear_snapshot() is called. + */ +PgStat_StatTabEntry * +pgstat_fetch_stat_tabentry_extended(PgStat_StatDBEntry *dbent, Oid reloid= ) +{ + /* context for snapshot_statentry */ + static pgstat_snapshot_param param =3D + { + .hash_name =3D "table stats snapshot hash", + .hash_entsize =3D sizeof(PgStat_StatTabEntry), + .dsh_handle =3D DSM_HANDLE_INVALID, + .dsh_params =3D &dsh_tblparams, + .hash =3D NULL, + .dshash =3D NULL + }; + + /* should be called from backends */ + Assert(IsUnderPostmaster); + + /* set target shared hash */ + param.dsh_handle =3D dbent->tables; + + /* tell snapshot_statentry what variables to use */ + param.hash =3D &dbent->snapshot_tables; + param.dshash =3D &dbent->dshash_tables; + + return (PgStat_StatTabEntry *) + snapshot_statentry(¶m, reloid); +} + = /* ---------- * pgstat_fetch_stat_funcentry() - @@ -2513,21 +2944,90 @@ pgstat_fetch_stat_funcentry(Oid func_id) PgStat_StatDBEntry *dbentry; PgStat_StatFuncEntry *funcentry =3D NULL; = - /* load the stats file if needed */ - backend_read_statsfile(); - - /* Lookup our database, then find the requested function. */ + /* Lookup our database, then find the requested function */ dbentry =3D pgstat_fetch_stat_dbentry(MyDatabaseId); - if (dbentry !=3D NULL && dbentry->functions !=3D NULL) - { - funcentry =3D (PgStat_StatFuncEntry *) hash_search(dbentry->functions, - (void *) &func_id, - HASH_FIND, NULL); - } + if (dbentry =3D=3D NULL) + return NULL; + + funcentry =3D pgstat_fetch_stat_funcentry_extended(dbentry, func_id); = return funcentry; } = +/* ---------- + * pgstat_fetch_stat_funcentry_extended() - + * + * Find function stats entry on backends. The returned entries are cached + * until transaction end or pgstat_clear_snapshot() is called. + * + * dbent is type of (PgStat_StatDBEntry *) but it's body must be an + * PgSTat_StatDBEntry returned from pgstat_fetch_stat_dbentry(). + */ +static PgStat_StatFuncEntry * +pgstat_fetch_stat_funcentry_extended(PgStat_StatDBEntry *dbent, Oid funci= d) +{ + /* context for snapshot_statentry */ + static pgstat_snapshot_param param =3D + { + .hash_name =3D "function stats snapshot hash", + .hash_entsize =3D sizeof(PgStat_StatFuncEntry), + .dsh_handle =3D DSM_HANDLE_INVALID, + .dsh_params =3D &dsh_funcparams, + .hash =3D NULL, + .dshash =3D NULL + }; + + /* should be called from backends */ + Assert(IsUnderPostmaster); + + if (dbent->functions =3D=3D DSM_HANDLE_INVALID) + return NULL; + + /* set target shared hash */ + param.dsh_handle =3D dbent->functions; + + /* tell snapshot_statentry what variables to use */ + param.hash =3D &dbent->snapshot_functions; + param.dshash =3D &dbent->dshash_functions; + + return (PgStat_StatFuncEntry *) + snapshot_statentry(¶m, funcid); +} + +/* + * pgstat_snapshot_global_stats() - + * + * Makes a snapshot of global stats if not done yet. They will be kept u= ntil + * subsequent call of pgstat_clear_snapshot() or the end of the current + * memory context (typically TopTransactionContext). + */ +static void +pgstat_snapshot_global_stats(void) +{ + MemoryContext oldcontext; + + pgstat_attach_shared_stats(); + + /* Nothing to do if already done */ + if (global_snapshot_is_valid) + return; + + oldcontext =3D MemoryContextSwitchTo(pgStatSnapshotContext); + + LWLockAcquire(StatsLock, LW_SHARED); + memcpy(&snapshot_globalStats, shared_globalStats, + sizeof(PgStat_GlobalStats)); + + memcpy(&snapshot_archiverStats, shared_archiverStats, + sizeof(PgStat_ArchiverStats)); + LWLockRelease(StatsLock); + + global_snapshot_is_valid =3D true; + + MemoryContextSwitchTo(oldcontext); + + return; +} = /* ---------- * pgstat_fetch_stat_beentry() - @@ -2599,9 +3099,10 @@ pgstat_fetch_stat_numbackends(void) PgStat_ArchiverStats * pgstat_fetch_stat_archiver(void) { - backend_read_statsfile(); + /* If not done for this transaction, take a stats snapshot */ + pgstat_snapshot_global_stats(); = - return &archiverStats; + return &snapshot_archiverStats; } = = @@ -2616,9 +3117,10 @@ pgstat_fetch_stat_archiver(void) PgStat_GlobalStats * pgstat_fetch_global(void) { - backend_read_statsfile(); + /* If not done for this transaction, take a stats snapshot */ + pgstat_snapshot_global_stats(); = - return &globalStats; + return &snapshot_globalStats; } = = @@ -2832,8 +3334,8 @@ pgstat_initialize(void) MyBEEntry =3D &BackendStatusArray[MaxBackends + MyAuxProcType]; } = - /* Set up a process-exit hook to clean up */ - on_shmem_exit(pgstat_beshutdown_hook, 0); + /* need to be called before dsm shutodwn */ + before_shmem_exit(pgstat_beshutdown_hook, 0); } = /* ---------- @@ -2931,7 +3433,7 @@ pgstat_bestart(void) lbeentry.st_backendType =3D B_STARTUP; break; case ArchiverProcess: - beentry->st_backendType =3D B_ARCHIVER; + lbeentry.st_backendType =3D B_ARCHIVER; break; case BgWriterProcess: lbeentry.st_backendType =3D B_BG_WRITER; @@ -3067,6 +3569,10 @@ pgstat_bestart(void) /* Update app name to current GUC setting */ if (application_name) pgstat_report_appname(application_name); + + + /* attach shared database stats area */ + pgstat_attach_shared_stats(); } = /* @@ -3102,6 +3608,8 @@ pgstat_beshutdown_hook(int code, Datum arg) beentry->st_procpid =3D 0; /* mark invalid */ = PGSTAT_END_WRITE_ACTIVITY(beentry); + + pgstat_detach_shared_stats(true); } = = @@ -3362,7 +3870,8 @@ pgstat_read_current_status(void) #endif int i; = - Assert(!pgStatRunningInCollector); + Assert(IsUnderPostmaster); + if (localBackendStatusTable) return; /* already done */ = @@ -3657,9 +4166,6 @@ pgstat_get_wait_activity(WaitEventActivity w) case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN: event_name =3D "LogicalLauncherMain"; break; - case WAIT_EVENT_PGSTAT_MAIN: - event_name =3D "PgStatMain"; - break; case WAIT_EVENT_RECOVERY_WAL_ALL: event_name =3D "RecoveryWalAll"; break; @@ -4322,75 +4828,43 @@ pgstat_get_backend_desc(BackendType backendType) * ------------------------------------------------------------ */ = - /* ---------- - * pgstat_setheader() - + * pgstat_send_archiver() - * - * Set common header fields in a statistics message + * Report archiver statistics * ---------- */ -static void -pgstat_setheader(PgStat_MsgHdr *hdr, StatMsgType mtype) +void +pgstat_send_archiver(const char *xlog, bool failed) { - hdr->m_type =3D mtype; -} + TimestampTz now =3D GetCurrentTimestamp(); = - -/* ---------- - * pgstat_send() - - * - * Send out one statistics message to the collector - * ---------- - */ -static void -pgstat_send(void *msg, int len) -{ - int rc; - - if (pgStatSock =3D=3D PGINVALID_SOCKET) - return; - - ((PgStat_MsgHdr *) msg)->m_size =3D len; - - /* We'll retry after EINTR, but ignore all other failures */ - do - { - rc =3D send(pgStatSock, msg, len, 0); - } while (rc < 0 && errno =3D=3D EINTR); - -#ifdef USE_ASSERT_CHECKING - /* In debug builds, log send failures ... */ - if (rc < 0) - elog(LOG, "could not send to statistics collector: %m"); -#endif -} - -/* ---------- - * pgstat_send_archiver() - - * - * Tell the collector about the WAL file that we successfully - * archived or failed to archive. - * ---------- - */ -void -pgstat_send_archiver(const char *xlog, bool failed) -{ - PgStat_MsgArchiver msg; - - /* - * Prepare and send the message - */ - pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_ARCHIVER); - msg.m_failed =3D failed; - StrNCpy(msg.m_xlog, xlog, sizeof(msg.m_xlog)); - msg.m_timestamp =3D GetCurrentTimestamp(); - pgstat_send(&msg, sizeof(msg)); -} + if (failed) + { + /* Failed archival attempt */ + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + ++shared_archiverStats->failed_count; + memcpy(shared_archiverStats->last_failed_wal, xlog, + sizeof(shared_archiverStats->last_failed_wal)); + shared_archiverStats->last_failed_timestamp =3D now; + LWLockRelease(StatsLock); + } + else + { + /* Successful archival operation */ + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + ++shared_archiverStats->archived_count; + memcpy(shared_archiverStats->last_archived_wal, xlog, + sizeof(shared_archiverStats->last_archived_wal)); + shared_archiverStats->last_archived_timestamp =3D now; + LWLockRelease(StatsLock); + } +} = /* ---------- * pgstat_send_bgwriter() - * - * Send bgwriter statistics to the collector + * Report bgwriter statistics * ---------- */ void @@ -4399,6 +4873,8 @@ pgstat_send_bgwriter(void) /* We assume this initializes to zeroes */ static const PgStat_MsgBgWriter all_zeroes; = + PgStat_MsgBgWriter *s =3D &BgWriterStats; + /* * This function can be called even if nothing at all has happened. In * this case, avoid sending a completely empty message to the stats @@ -4407,11 +4883,18 @@ pgstat_send_bgwriter(void) if (memcmp(&BgWriterStats, &all_zeroes, sizeof(PgStat_MsgBgWriter)) =3D=3D= 0) return; = - /* - * Prepare and send the message - */ - pgstat_setheader(&BgWriterStats.m_hdr, PGSTAT_MTYPE_BGWRITER); - pgstat_send(&BgWriterStats, sizeof(BgWriterStats)); + LWLockAcquire(StatsLock, LW_EXCLUSIVE); + shared_globalStats->timed_checkpoints +=3D s->m_timed_checkpoints; + shared_globalStats->requested_checkpoints +=3D s->m_requested_checkpoint= s; + shared_globalStats->checkpoint_write_time +=3D s->m_checkpoint_write_tim= e; + shared_globalStats->checkpoint_sync_time +=3D s->m_checkpoint_sync_time; + shared_globalStats->buf_written_checkpoints +=3D s->m_buf_written_checkp= oints; + shared_globalStats->buf_written_clean +=3D s->m_buf_written_clean; + shared_globalStats->maxwritten_clean +=3D s->m_maxwritten_clean; + shared_globalStats->buf_written_backend +=3D s->m_buf_written_backend; + shared_globalStats->buf_fsync_backend +=3D s->m_buf_fsync_backend; + shared_globalStats->buf_alloc +=3D s->m_buf_alloc; + LWLockRelease(StatsLock); = /* * Clear out the statistics buffer, so it can be re-used. @@ -4420,276 +4903,164 @@ pgstat_send_bgwriter(void) } = = -/* ---------- - * PgstatCollectorMain() - - * - * Start up the statistics collector process. This is the body of the - * postmaster child process. +/* + * Pin and Unpin dbentry. * - * The argc/argv parameters are valid only in EXEC_BACKEND case. - * ---------- + * To keep less memory usage, and for speed, counters are by recreation o= f + * dshash instead of removing entries one-by-one keeping whole-dshash loc= k. On + * the other hand dshash cannot be destroyed until all referrers have gon= e. As + * the result, other backend may be kept waiting the counter reset for no= t a + * short time. We isolate the hashes under destruction as another generat= ion, + * which means no longer used but cannot be removed yet. + + * When we start accessing hashes on a dbentry, call pin_hashes() and acq= uire + * the current "generation". Unlock removes the older generation's hashes= when + * all refers have gone. */ -NON_EXEC_STATIC void -PgstatCollectorMain(int argc, char *argv[]) +static int +pin_hashes(PgStat_StatDBEntry *dbentry) { - int len; - PgStat_Msg msg; - int wr; + int generation; = - /* - * Ignore all signals usually bound to some action in the postmaster, - * except SIGHUP and SIGQUIT. Note we don't need a SIGUSR1 handler to - * support latch operations, because we only use a local latch. - */ - pqsignal(SIGHUP, SignalHandlerForConfigReload); - pqsignal(SIGINT, SIG_IGN); - pqsignal(SIGTERM, SIG_IGN); - pqsignal(SIGQUIT, SignalHandlerForShutdownRequest); - pqsignal(SIGALRM, SIG_IGN); - pqsignal(SIGPIPE, SIG_IGN); - pqsignal(SIGUSR1, SIG_IGN); - pqsignal(SIGUSR2, SIG_IGN); - /* Reset some signals that are accepted by postmaster but not here */ - pqsignal(SIGCHLD, SIG_DFL); - PG_SETMASK(&UnBlockSig); + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); + dbentry->refcnt++; + generation =3D dbentry->generation; + LWLockRelease(&dbentry->lock); = - /* - * Identify myself via ps - */ - init_ps_display("stats collector", "", "", ""); + dshash_release_lock(pgStatDBHash, dbentry); = - /* - * Read in existing stats files or initialize the stats to zero. - */ - pgStatRunningInCollector =3D true; - pgStatDBHash =3D pgstat_read_statsfiles(InvalidOid, true, true); + return generation; +} + +/* + * Unpin hashes in dbentry. If given generation is isolated, destroy it a= fter + * all referrers has gone. Otherwise just decrease reference count then r= eturn. + */ +static void +unpin_hashes(PgStat_StatDBEntry *dbentry, int generation) +{ + dshash_table *tables; + dshash_table *funcs =3D NULL; + + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); + + /* using current generation, just decrease refcount */ + if (dbentry->generation =3D=3D generation) + { + dbentry->refcnt--; + LWLockRelease(&dbentry->lock); + return; + } = /* - * Loop to process messages until we get SIGQUIT or detect ungraceful - * death of our parent postmaster. - * - * For performance reasons, we don't want to do ResetLatch/WaitLatch aft= er - * every message; instead, do that only after a recv() fails to obtain a - * message. (This effectively means that if backends are sending us stu= ff - * like mad, we won't notice postmaster death until things slack off a - * bit; which seems fine.) To do that, we have an inner loop that - * iterates as long as recv() succeeds. We do check ConfigReloadPending - * inside the inner loop, which means that such interrupts will get - * serviced but the latch won't get cleared until next time there is a - * break in the action. + * It is isolated, waiting for all referrers to end. */ - for (;;) + Assert(dbentry->generation =3D=3D generation + 1); + + if (--dbentry->prev_refcnt > 0) { - /* Clear any already-pending wakeups */ - ResetLatch(MyLatch); + LWLockRelease(&dbentry->lock); + return; + } = - /* - * Quit if we get SIGQUIT from the postmaster. - */ - if (ShutdownRequestPending) - break; + /* no referrer remains, remove the hashes */ + tables =3D dshash_attach(area, &dsh_tblparams, dbentry->prev_tables, 0); + if (dbentry->prev_functions !=3D DSM_HANDLE_INVALID) + funcs =3D dshash_attach(area, &dsh_funcparams, + dbentry->prev_functions, 0); = - /* - * Inner loop iterates as long as we keep getting messages, or until - * ShutdownRequestPending becomes set. - */ - while (!ShutdownRequestPending) - { - /* - * Reload configuration if we got SIGHUP from the postmaster. - */ - if (ConfigReloadPending) - { - ConfigReloadPending =3D false; - ProcessConfigFile(PGC_SIGHUP); - } + dbentry->prev_tables =3D DSM_HANDLE_INVALID; + dbentry->prev_functions =3D DSM_HANDLE_INVALID; = - /* - * Write the stats file(s) if a new request has arrived that is - * not satisfied by existing file(s). - */ - if (pgstat_write_statsfile_needed()) - pgstat_write_statsfiles(false, false); + /* release the entry immediately */ + LWLockRelease(&dbentry->lock); = - /* - * Try to receive and process a message. This will not block, - * since the socket is set to non-blocking mode. - * - * XXX On Windows, we have to force pgwin32_recv to cooperate, - * despite the previous use of pg_set_noblock() on the socket. - * This is extremely broken and should be fixed someday. - */ -#ifdef WIN32 - pgwin32_noblock =3D 1; -#endif + dshash_destroy(tables); + if (funcs) + dshash_destroy(funcs); = - len =3D recv(pgStatSock, (char *) &msg, - sizeof(PgStat_Msg), 0); + return; +} = -#ifdef WIN32 - pgwin32_noblock =3D 0; -#endif +/* + * attach and return the specified generation of table hash + * Returns NULL on lock failure. + */ +static dshash_table * +attach_table_hash(PgStat_StatDBEntry *dbent, int gen) +{ + dshash_table *ret; = - if (len < 0) - { - if (errno =3D=3D EAGAIN || errno =3D=3D EWOULDBLOCK || errno =3D=3D E= INTR) - break; /* out of inner loop */ - ereport(ERROR, - (errcode_for_socket_access(), - errmsg("could not read statistics message: %m"))); - } + LWLockAcquire(&dbent->lock, LW_EXCLUSIVE); = - /* - * We ignore messages that are smaller than our common header - */ - if (len < sizeof(PgStat_MsgHdr)) - continue; + if (dbent->generation =3D=3D gen) + ret =3D dshash_attach(area, &dsh_tblparams, dbent->tables, 0); + else + { + Assert (dbent->generation =3D=3D gen + 1); + Assert (dbent->prev_tables !=3D DSM_HANDLE_INVALID); + ret =3D dshash_attach(area, &dsh_tblparams, dbent->prev_tables, 0); + } + LWLockRelease(&dbent->lock); = - /* - * The received length must match the length in the header - */ - if (msg.msg_hdr.m_size !=3D len) - continue; + return ret; +} = - /* - * O.K. - we accept this message. Process it. - */ - switch (msg.msg_hdr.m_type) - { - case PGSTAT_MTYPE_DUMMY: - break; - - case PGSTAT_MTYPE_INQUIRY: - pgstat_recv_inquiry(&msg.msg_inquiry, len); - break; - - case PGSTAT_MTYPE_TABSTAT: - pgstat_recv_tabstat(&msg.msg_tabstat, len); - break; - - case PGSTAT_MTYPE_TABPURGE: - pgstat_recv_tabpurge(&msg.msg_tabpurge, len); - break; - - case PGSTAT_MTYPE_DROPDB: - pgstat_recv_dropdb(&msg.msg_dropdb, len); - break; - - case PGSTAT_MTYPE_RESETCOUNTER: - pgstat_recv_resetcounter(&msg.msg_resetcounter, len); - break; - - case PGSTAT_MTYPE_RESETSHAREDCOUNTER: - pgstat_recv_resetsharedcounter(&msg.msg_resetsharedcounter, - len); - break; - - case PGSTAT_MTYPE_RESETSINGLECOUNTER: - pgstat_recv_resetsinglecounter(&msg.msg_resetsinglecounter, - len); - break; - - case PGSTAT_MTYPE_AUTOVAC_START: - pgstat_recv_autovac(&msg.msg_autovacuum_start, len); - break; - - case PGSTAT_MTYPE_VACUUM: - pgstat_recv_vacuum(&msg.msg_vacuum, len); - break; - - case PGSTAT_MTYPE_ANALYZE: - pgstat_recv_analyze(&msg.msg_analyze, len); - break; - - case PGSTAT_MTYPE_ARCHIVER: - pgstat_recv_archiver(&msg.msg_archiver, len); - break; - - case PGSTAT_MTYPE_BGWRITER: - pgstat_recv_bgwriter(&msg.msg_bgwriter, len); - break; - - case PGSTAT_MTYPE_FUNCSTAT: - pgstat_recv_funcstat(&msg.msg_funcstat, len); - break; - - case PGSTAT_MTYPE_FUNCPURGE: - pgstat_recv_funcpurge(&msg.msg_funcpurge, len); - break; - - case PGSTAT_MTYPE_RECOVERYCONFLICT: - pgstat_recv_recoveryconflict(&msg.msg_recoveryconflict, - len); - break; - - case PGSTAT_MTYPE_DEADLOCK: - pgstat_recv_deadlock(&msg.msg_deadlock, len); - break; - - case PGSTAT_MTYPE_TEMPFILE: - pgstat_recv_tempfile(&msg.msg_tempfile, len); - break; - - case PGSTAT_MTYPE_CHECKSUMFAILURE: - pgstat_recv_checksum_failure(&msg.msg_checksumfailure, - len); - break; - - default: - break; - } - } /* end of inner message-processing loop */ - - /* Sleep until there's something to do */ -#ifndef WIN32 - wr =3D WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_SOCKET_READABLE, - pgStatSock, -1L, - WAIT_EVENT_PGSTAT_MAIN); -#else +/* attach and return the specified generation of function hash */ +static dshash_table * +attach_function_hash(PgStat_StatDBEntry *dbent, int gen) +{ + dshash_table *ret =3D NULL; = - /* - * Windows, at least in its Windows Server 2003 R2 incarnation, - * sometimes loses FD_READ events. Waking up and retrying the recv() - * fixes that, so don't sleep indefinitely. This is a crock of the - * first water, but until somebody wants to debug exactly what's - * happening there, this is the best we can do. The two-second - * timeout matches our pre-9.2 behavior, and needs to be short enough - * to not provoke "using stale statistics" complaints from - * backend_read_statsfile. - */ - wr =3D WaitLatchOrSocket(MyLatch, - WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_SOCKET_READABLE | WL_TI= MEOUT, - pgStatSock, - 2 * 1000L /* msec */ , - WAIT_EVENT_PGSTAT_MAIN); -#endif = - /* - * Emergency bailout if postmaster has died. This is to avoid the - * necessity for manual cleanup of all postmaster children. - */ - if (wr & WL_POSTMASTER_DEATH) - break; - } /* end of outer loop */ + LWLockAcquire(&dbent->lock, LW_EXCLUSIVE); = - /* - * Save the final stats to reuse at next startup. - */ - pgstat_write_statsfiles(true, true); + if (dbent->generation =3D=3D gen) + { + if (dbent->functions =3D=3D DSM_HANDLE_INVALID) + { + dshash_table *funchash =3D + dshash_create(area, &dsh_funcparams, 0); + dbent->functions =3D dshash_get_hash_table_handle(funchash); + + ret =3D funchash; + } + else + ret =3D dshash_attach(area, &dsh_funcparams, dbent->functions, 0); + } + /* don't bother creating useless hash */ + + LWLockRelease(&dbent->lock); + + return ret; +} = - exit(0); +static void +init_dbentry(PgStat_StatDBEntry *dbentry) +{ + LWLockInitialize(&dbentry->lock, LWTRANCHE_STATS); + dbentry->generation =3D 0; + dbentry->refcnt =3D 0; + dbentry->prev_refcnt =3D 0; + dbentry->tables =3D DSM_HANDLE_INVALID; + dbentry->prev_tables =3D DSM_HANDLE_INVALID; + dbentry->functions =3D DSM_HANDLE_INVALID; + dbentry->prev_functions =3D DSM_HANDLE_INVALID; } = /* * Subroutine to clear stats in a database entry * - * Tables and functions hashes are initialized to empty. + * Reset all counters in the dbentry. Tables and functions dshashes are + * destroyed. If any backend is pinning this dbentry, the current dshash= es + * are stashed out to the previous "generation" to wait for all accessors= are + * gone. If the previous generation is already occupied, the current dsha= shes + * are so fresh that they doesn't need to be cleared. */ static void reset_dbentry_counters(PgStat_StatDBEntry *dbentry) { - HASHCTL hash_ctl; + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); = dbentry->n_xact_commit =3D 0; dbentry->n_xact_rollback =3D 0; @@ -4714,130 +5085,118 @@ reset_dbentry_counters(PgStat_StatDBEntry *dbent= ry) dbentry->n_block_read_time =3D 0; dbentry->n_block_write_time =3D 0; = - dbentry->stat_reset_timestamp =3D GetCurrentTimestamp(); - dbentry->stats_timestamp =3D 0; - - memset(&hash_ctl, 0, sizeof(hash_ctl)); - hash_ctl.keysize =3D sizeof(Oid); - hash_ctl.entrysize =3D sizeof(PgStat_StatTabEntry); - dbentry->tables =3D hash_create("Per-database table", - PGSTAT_TAB_HASH_SIZE, - &hash_ctl, - HASH_ELEM | HASH_BLOBS); - - hash_ctl.keysize =3D sizeof(Oid); - hash_ctl.entrysize =3D sizeof(PgStat_StatFuncEntry); - dbentry->functions =3D hash_create("Per-database function", - PGSTAT_FUNCTION_HASH_SIZE, - &hash_ctl, - HASH_ELEM | HASH_BLOBS); -} + if (dbentry->refcnt =3D=3D 0) + { + /* + * No one is referring to the current hash. It's very costly to remove + * entries in dshash individually so just destroy the whole. If + * someone pined this entry just after, pin_hashes() returns the + * current generation and attach will happen after the following + * LWLock released. + */ + dshash_table *tbl; = -/* - * Lookup the hash table entry for the specified database. If no hash - * table entry exists, initialize it, if the create parameter is true. - * Else, return NULL. - */ -static PgStat_StatDBEntry * -pgstat_get_db_entry(Oid databaseid, bool create) -{ - PgStat_StatDBEntry *result; - bool found; - HASHACTION action =3D (create ? HASH_ENTER : HASH_FIND); + if (dbentry->tables !=3D DSM_HANDLE_INVALID) + { + tbl =3D dshash_attach(area, &dsh_tblparams, dbentry->tables, 0); + dshash_destroy(tbl); + dbentry->tables =3D DSM_HANDLE_INVALID; + } + if (dbentry->functions !=3D DSM_HANDLE_INVALID) + { + tbl =3D dshash_attach(area, &dsh_funcparams, dbentry->functions, 0); + dshash_destroy(tbl); + dbentry->functions =3D DSM_HANDLE_INVALID; + } + } + else if (dbentry->prev_refcnt =3D=3D 0) + { + /* + * Someone is still referring to the current hash and previous slot is + * vacant. Stash out the current hash to the previous slot. + */ + dbentry->prev_refcnt =3D dbentry->refcnt; + dbentry->prev_tables =3D dbentry->tables; + dbentry->prev_functions =3D dbentry->functions; + dbentry->refcnt =3D 0; + dbentry->tables =3D DSM_HANDLE_INVALID; + dbentry->functions =3D DSM_HANDLE_INVALID; + dbentry->generation++; + } + else + { + Assert(dbentry->prev_refcnt > 0 && dbentry->refcnt > 0); + /* + * If we get here, we just have got another reset request and the old + * hashes are waiting to all referrers to be released. It must be + * quite a short time so we can just ignore this request. + * + * As the side effect, the resetter can see non-zero values before + * anyone updates them but it's not distinctive with someone updated + * them before reading. + */ + } = - /* Lookup or create the hash table entry for this database */ - result =3D (PgStat_StatDBEntry *) hash_search(pgStatDBHash, - &databaseid, - action, &found); + /* Create new table hash if not exists */ + if (dbentry->tables =3D=3D DSM_HANDLE_INVALID) + { + dshash_table *tbl =3D dshash_create(area, &dsh_tblparams, 0); + dbentry->tables =3D dshash_get_hash_table_handle(tbl); + dshash_detach(tbl); + } = - if (!create && !found) - return NULL; + /* Create new function hash if not exists and needed. */ + if (dbentry->functions =3D=3D DSM_HANDLE_INVALID && + pgstat_track_functions !=3D TRACK_FUNC_OFF) + { + dshash_table *tbl =3D dshash_create(area, &dsh_funcparams, 0); + dbentry->functions =3D dshash_get_hash_table_handle(tbl); + dshash_detach(tbl); + } = - /* - * If not found, initialize the new one. This creates empty hash tables - * for tables and functions, too. - */ - if (!found) - reset_dbentry_counters(result); + dbentry->stat_reset_timestamp =3D GetCurrentTimestamp(); = - return result; + LWLockRelease(&dbentry->lock); } = - /* - * Lookup the hash table entry for the specified table. If no hash - * table entry exists, initialize it, if the create parameter is true. - * Else, return NULL. + * Create the filename for a DB stat file; filename is output parameter p= oints + * to a character buffer of length len. */ -static PgStat_StatTabEntry * -pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, Oid tableoid, bool crea= te) +static void +get_dbstat_filename(bool tempname, Oid databaseid, char *filename, int le= n) { - PgStat_StatTabEntry *result; - bool found; - HASHACTION action =3D (create ? HASH_ENTER : HASH_FIND); - - /* Lookup or create the hash table entry for this table */ - result =3D (PgStat_StatTabEntry *) hash_search(dbentry->tables, - &tableoid, - action, &found); - - if (!create && !found) - return NULL; - - /* If not found, initialize the new one. */ - if (!found) - { - result->numscans =3D 0; - result->tuples_returned =3D 0; - result->tuples_fetched =3D 0; - result->tuples_inserted =3D 0; - result->tuples_updated =3D 0; - result->tuples_deleted =3D 0; - result->tuples_hot_updated =3D 0; - result->n_live_tuples =3D 0; - result->n_dead_tuples =3D 0; - result->changes_since_analyze =3D 0; - result->blocks_fetched =3D 0; - result->blocks_hit =3D 0; - result->vacuum_timestamp =3D 0; - result->vacuum_count =3D 0; - result->autovac_vacuum_timestamp =3D 0; - result->autovac_vacuum_count =3D 0; - result->analyze_timestamp =3D 0; - result->analyze_count =3D 0; - result->autovac_analyze_timestamp =3D 0; - result->autovac_analyze_count =3D 0; - } + int printed; = - return result; + /* NB -- pgstat_reset_remove_files knows about the pattern this uses */ + printed =3D snprintf(filename, len, "%s/db_%u.%s", + PGSTAT_STAT_PERMANENT_DIRECTORY, + databaseid, + tempname ? "tmp" : "stat"); + if (printed >=3D len) + elog(ERROR, "overlength pgstat path"); } = - /* ---------- * pgstat_write_statsfiles() - - * Write the global statistics file, as well as requested DB files. - * - * 'permanent' specifies writing to the permanent files not temporary one= s. - * When true (happens only when the collector is shutting down), also rem= ove - * the temporary files so that backends starting up under a new postmaste= r - * can't read old data before the new collector is ready. - * - * When 'allDbs' is false, only the requested databases (listed in - * pending_write_requests) will be written; otherwise, all databases - * will be written. + * Write the global statistics file, as well as DB files. * ---------- */ -static void -pgstat_write_statsfiles(bool permanent, bool allDbs) +void +pgstat_write_statsfiles(void) { - HASH_SEQ_STATUS hstat; + dshash_seq_status hstat; PgStat_StatDBEntry *dbentry; FILE *fpout; int32 format_id; - const char *tmpfile =3D permanent ? PGSTAT_STAT_PERMANENT_TMPFILE : pgst= at_stat_tmpname; - const char *statfile =3D permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pg= stat_stat_filename; + const char *tmpfile =3D PGSTAT_STAT_PERMANENT_TMPFILE; + const char *statfile =3D PGSTAT_STAT_PERMANENT_FILENAME; int rc; = + /* stats is not initialized yet. just return. */ + if (StatsShmem->stats_dsa_handle =3D=3D DSM_HANDLE_INVALID) + return; + elog(DEBUG2, "writing stats file \"%s\"", statfile); = /* @@ -4856,7 +5215,7 @@ pgstat_write_statsfiles(bool permanent, bool allDbs) /* * Set the timestamp of the stats file. */ - globalStats.stats_timestamp =3D GetCurrentTimestamp(); + shared_globalStats->stats_timestamp =3D GetCurrentTimestamp(); = /* * Write the file header --- currently just a format ID. @@ -4868,39 +5227,37 @@ pgstat_write_statsfiles(bool permanent, bool allDb= s) /* * Write global stats struct */ - rc =3D fwrite(&globalStats, sizeof(globalStats), 1, fpout); + rc =3D fwrite(shared_globalStats, sizeof(*shared_globalStats), 1, fpout)= ; (void) rc; /* we'll check for error with ferror */ = /* * Write archiver stats struct */ - rc =3D fwrite(&archiverStats, sizeof(archiverStats), 1, fpout); + rc =3D fwrite(shared_archiverStats, sizeof(*shared_archiverStats), 1, fp= out); (void) rc; /* we'll check for error with ferror */ = /* * Walk through the database table. */ - hash_seq_init(&hstat, pgStatDBHash); - while ((dbentry =3D (PgStat_StatDBEntry *) hash_seq_search(&hstat)) !=3D= NULL) + dshash_seq_init(&hstat, pgStatDBHash, false, false); + while ((dbentry =3D (PgStat_StatDBEntry *) dshash_seq_next(&hstat)) !=3D= NULL) { /* * Write out the table and function stats for this DB into the * appropriate per-DB stat file, if required. */ - if (allDbs || pgstat_db_requested(dbentry->databaseid)) - { - /* Make DB's timestamp consistent with the global stats */ - dbentry->stats_timestamp =3D globalStats.stats_timestamp; + /* Make DB's timestamp consistent with the global stats */ + dbentry->stats_timestamp =3D shared_globalStats->stats_timestamp; = - pgstat_write_db_statsfile(dbentry, permanent); - } + pgstat_write_pgStatDBHashfile(dbentry); = /* * Write out the DB entry. We don't write the tables or functions * pointers, since they're of no use to any other process. */ fputc('D', fpout); - rc =3D fwrite(dbentry, offsetof(PgStat_StatDBEntry, tables), 1, fpout); + rc =3D fwrite(dbentry, + offsetof(PgStat_StatDBEntry, generation), 1, fpout); (void) rc; /* we'll check for error with ferror */ } = @@ -4936,53 +5293,18 @@ pgstat_write_statsfiles(bool permanent, bool allDb= s) tmpfile, statfile))); unlink(tmpfile); } - - if (permanent) - unlink(pgstat_stat_filename); - - /* - * Now throw away the list of requests. Note that requests sent after w= e - * started the write are still waiting on the network socket. - */ - list_free(pending_write_requests); - pending_write_requests =3D NIL; -} - -/* - * return the filename for a DB stat file; filename is the output buffer, - * of length len. - */ -static void -get_dbstat_filename(bool permanent, bool tempname, Oid databaseid, - char *filename, int len) -{ - int printed; - - /* NB -- pgstat_reset_remove_files knows about the pattern this uses */ - printed =3D snprintf(filename, len, "%s/db_%u.%s", - permanent ? PGSTAT_STAT_PERMANENT_DIRECTORY : - pgstat_stat_directory, - databaseid, - tempname ? "tmp" : "stat"); - if (printed >=3D len) - elog(ERROR, "overlength pgstat path"); } = /* ---------- - * pgstat_write_db_statsfile() - + * pgstat_write_pgStatDBHashfile() - * Write the stat file for a single database. - * - * If writing to the permanent file (happens when the collector is - * shutting down only), remove the temporary file so that backends - * starting up under a new postmaster can't read the old data before - * the new collector is ready. * ---------- */ static void -pgstat_write_db_statsfile(PgStat_StatDBEntry *dbentry, bool permanent) +pgstat_write_pgStatDBHashfile(PgStat_StatDBEntry *dbentry) { - HASH_SEQ_STATUS tstat; - HASH_SEQ_STATUS fstat; + dshash_seq_status tstat; + dshash_seq_status fstat; PgStat_StatTabEntry *tabentry; PgStat_StatFuncEntry *funcentry; FILE *fpout; @@ -4991,9 +5313,10 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dbent= ry, bool permanent) int rc; char tmpfile[MAXPGPATH]; char statfile[MAXPGPATH]; + dshash_table *tbl; = - get_dbstat_filename(permanent, true, dbid, tmpfile, MAXPGPATH); - get_dbstat_filename(permanent, false, dbid, statfile, MAXPGPATH); + get_dbstat_filename(true, dbid, tmpfile, MAXPGPATH); + get_dbstat_filename(false, dbid, statfile, MAXPGPATH); = elog(DEBUG2, "writing stats file \"%s\"", statfile); = @@ -5020,23 +5343,30 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dben= try, bool permanent) /* * Walk through the database's access stats per table. */ - hash_seq_init(&tstat, dbentry->tables); - while ((tabentry =3D (PgStat_StatTabEntry *) hash_seq_search(&tstat)) !=3D= NULL) + tbl =3D dshash_attach(area, &dsh_tblparams, dbentry->tables, 0); + dshash_seq_init(&tstat, tbl, false, false); + while ((tabentry =3D (PgStat_StatTabEntry *) dshash_seq_next(&tstat)) !=3D= NULL) { fputc('T', fpout); rc =3D fwrite(tabentry, sizeof(PgStat_StatTabEntry), 1, fpout); (void) rc; /* we'll check for error with ferror */ } + dshash_detach(tbl); = /* * Walk through the database's function stats table. */ - hash_seq_init(&fstat, dbentry->functions); - while ((funcentry =3D (PgStat_StatFuncEntry *) hash_seq_search(&fstat)) = !=3D NULL) + if (dbentry->functions !=3D DSM_HANDLE_INVALID) { - fputc('F', fpout); - rc =3D fwrite(funcentry, sizeof(PgStat_StatFuncEntry), 1, fpout); - (void) rc; /* we'll check for error with ferror */ + tbl =3D dshash_attach(area, &dsh_funcparams, dbentry->functions, 0); + dshash_seq_init(&fstat, tbl, false, false); + while ((funcentry =3D (PgStat_StatFuncEntry *) dshash_seq_next(&fstat))= !=3D NULL) + { + fputc('F', fpout); + rc =3D fwrite(funcentry, sizeof(PgStat_StatFuncEntry), 1, fpout); + (void) rc; /* we'll check for error with ferror */ + } + dshash_detach(tbl); } = /* @@ -5071,76 +5401,37 @@ pgstat_write_db_statsfile(PgStat_StatDBEntry *dben= try, bool permanent) tmpfile, statfile))); unlink(tmpfile); } - - if (permanent) - { - get_dbstat_filename(false, false, dbid, statfile, MAXPGPATH); - - elog(DEBUG2, "removing temporary stats file \"%s\"", statfile); - unlink(statfile); - } } = /* ---------- * pgstat_read_statsfiles() - * - * Reads in some existing statistics collector files and returns the - * databases hash table that is the top level of the data. - * - * If 'onlydb' is not InvalidOid, it means we only want data for that DB - * plus the shared catalogs ("DB 0"). We'll still populate the DB hash - * table for all databases, but we don't bother even creating table/funct= ion - * hash tables for other databases. + * Reads in existing statistics collector files into the shared stats has= h. * - * 'permanent' specifies reading from the permanent files not temporary o= nes. - * When true (happens only when the collector is starting up), remove the - * files after reading; the in-memory status is now authoritative, and th= e - * files would be out of date in case somebody else reads them. - * - * If a 'deep' read is requested, table/function stats are read, otherwis= e - * the table/function hash tables remain empty. * ---------- */ -static HTAB * -pgstat_read_statsfiles(Oid onlydb, bool permanent, bool deep) +void +pgstat_read_statsfiles(void) { PgStat_StatDBEntry *dbentry; PgStat_StatDBEntry dbbuf; - HASHCTL hash_ctl; - HTAB *dbhash; FILE *fpin; int32 format_id; bool found; - const char *statfile =3D permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pg= stat_stat_filename; + const char *statfile =3D PGSTAT_STAT_PERMANENT_FILENAME; + + /* shouldn't be called from postmaster */ + Assert(IsUnderPostmaster); + + elog(DEBUG2, "reading stats file \"%s\"", statfile); = /* - * The tables will live in pgStatLocalContext. + * Set the current timestamp (will be kept only in case we can't load an + * existing statsfile). */ - pgstat_setup_memcxt(); - - /* - * Create the DB hashtable - */ - memset(&hash_ctl, 0, sizeof(hash_ctl)); - hash_ctl.keysize =3D sizeof(Oid); - hash_ctl.entrysize =3D sizeof(PgStat_StatDBEntry); - hash_ctl.hcxt =3D pgStatLocalContext; - dbhash =3D hash_create("Databases hash", PGSTAT_DB_HASH_SIZE, &hash_ctl, - HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); - - /* - * Clear out global and archiver statistics so they start from zero in - * case we can't load an existing statsfile. - */ - memset(&globalStats, 0, sizeof(globalStats)); - memset(&archiverStats, 0, sizeof(archiverStats)); - - /* - * Set the current timestamp (will be kept only in case we can't load an - * existing statsfile). - */ - globalStats.stat_reset_timestamp =3D GetCurrentTimestamp(); - archiverStats.stat_reset_timestamp =3D globalStats.stat_reset_timestamp; + shared_globalStats->stat_reset_timestamp =3D GetCurrentTimestamp(); + shared_archiverStats->stat_reset_timestamp =3D + shared_globalStats->stat_reset_timestamp; = /* * Try to open the stats file. If it doesn't exist, the backends simply @@ -5154,11 +5445,11 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent,= bool deep) if ((fpin =3D AllocateFile(statfile, PG_BINARY_R)) =3D=3D NULL) { if (errno !=3D ENOENT) - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errcode_for_file_access(), errmsg("could not open statistics file \"%s\": %m", statfile))); - return dbhash; + return; } = /* @@ -5167,7 +5458,7 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent, b= ool deep) if (fread(&format_id, 1, sizeof(format_id), fpin) !=3D sizeof(format_id)= || format_id !=3D PGSTAT_FILE_FORMAT_ID) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } @@ -5175,32 +5466,24 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent,= bool deep) /* * Read global stats struct */ - if (fread(&globalStats, 1, sizeof(globalStats), fpin) !=3D sizeof(global= Stats)) + if (fread(shared_globalStats, 1, sizeof(*shared_globalStats), fpin) !=3D + sizeof(*shared_globalStats)) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); - memset(&globalStats, 0, sizeof(globalStats)); + MemSet(shared_globalStats, 0, sizeof(*shared_globalStats)); goto done; } = /* - * In the collector, disregard the timestamp we read from the permanent - * stats file; we should be willing to write a temp stats file immediate= ly - * upon the first request from any backend. This only matters if the ol= d - * file's timestamp is less than PGSTAT_STAT_INTERVAL ago, but that's no= t - * an unusual scenario. - */ - if (pgStatRunningInCollector) - globalStats.stats_timestamp =3D 0; - - /* * Read archiver stats struct */ - if (fread(&archiverStats, 1, sizeof(archiverStats), fpin) !=3D sizeof(ar= chiverStats)) + if (fread(shared_archiverStats, 1, sizeof(*shared_archiverStats), fpin) = !=3D + sizeof(*shared_archiverStats)) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); - memset(&archiverStats, 0, sizeof(archiverStats)); + MemSet(shared_archiverStats, 0, sizeof(*shared_archiverStats)); goto done; } = @@ -5217,10 +5500,10 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent,= bool deep) * follows. */ case 'D': - if (fread(&dbbuf, 1, offsetof(PgStat_StatDBEntry, tables), - fpin) !=3D offsetof(PgStat_StatDBEntry, tables)) + if (fread(&dbbuf, 1, offsetof(PgStat_StatDBEntry, generation), + fpin) !=3D offsetof(PgStat_StatDBEntry, generation)) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; @@ -5229,76 +5512,36 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent,= bool deep) /* * Add to the DB hash */ - dbentry =3D (PgStat_StatDBEntry *) hash_search(dbhash, - (void *) &dbbuf.databaseid, - HASH_ENTER, - &found); + dbentry =3D (PgStat_StatDBEntry *) + dshash_find_or_insert(pgStatDBHash, (void *) &dbbuf.databaseid, + &found); + + /* don't allow duplicate dbentries */ if (found) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + dshash_release_lock(pgStatDBHash, dbentry); + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = - memcpy(dbentry, &dbbuf, sizeof(PgStat_StatDBEntry)); - dbentry->tables =3D NULL; - dbentry->functions =3D NULL; - - /* - * In the collector, disregard the timestamp we read from the - * permanent stats file; we should be willing to write a temp - * stats file immediately upon the first request from any - * backend. - */ - if (pgStatRunningInCollector) - dbentry->stats_timestamp =3D 0; - - /* - * Don't create tables/functions hashtables for uninteresting - * databases. - */ - if (onlydb !=3D InvalidOid) - { - if (dbbuf.databaseid !=3D onlydb && - dbbuf.databaseid !=3D InvalidOid) - break; - } - - memset(&hash_ctl, 0, sizeof(hash_ctl)); - hash_ctl.keysize =3D sizeof(Oid); - hash_ctl.entrysize =3D sizeof(PgStat_StatTabEntry); - hash_ctl.hcxt =3D pgStatLocalContext; - dbentry->tables =3D hash_create("Per-database table", - PGSTAT_TAB_HASH_SIZE, - &hash_ctl, - HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); - - hash_ctl.keysize =3D sizeof(Oid); - hash_ctl.entrysize =3D sizeof(PgStat_StatFuncEntry); - hash_ctl.hcxt =3D pgStatLocalContext; - dbentry->functions =3D hash_create("Per-database function", - PGSTAT_FUNCTION_HASH_SIZE, - &hash_ctl, - HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + /* initialize the new shared entry */ + init_dbentry(dbentry); = - /* - * If requested, read the data from the database-specific - * file. Otherwise we just leave the hashtables empty. - */ - if (deep) - pgstat_read_db_statsfile(dbentry->databaseid, - dbentry->tables, - dbentry->functions, - permanent); + memcpy(dbentry, &dbbuf, + offsetof(PgStat_StatDBEntry, generation)); = + /* Read the data from the database-specific file. */ + pgstat_read_pgStatDBHashfile(dbentry); + dshash_release_lock(pgStatDBHash, dbentry); break; = case 'E': goto done; = default: - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; @@ -5308,45 +5551,35 @@ pgstat_read_statsfiles(Oid onlydb, bool permanent,= bool deep) done: FreeFile(fpin); = - /* If requested to read the permanent file, also get rid of it. */ - if (permanent) - { - elog(DEBUG2, "removing permanent stats file \"%s\"", statfile); - unlink(statfile); - } + elog(DEBUG2, "removing permanent stats file \"%s\"", statfile); + unlink(statfile); = - return dbhash; + return; } = = /* ---------- - * pgstat_read_db_statsfile() - - * - * Reads in the existing statistics collector file for the given database= , - * filling the passed-in tables and functions hash tables. + * pgstat_read_pgStatDBHashfile() - * - * As in pgstat_read_statsfiles, if the permanent file is requested, it i= s - * removed after reading. - * - * Note: this code has the ability to skip storing per-table or per-funct= ion - * data, if NULL is passed for the corresponding hashtable. That's not u= sed - * at the moment though. + * Reads in the at-rest statistics file and create shared statistics + * tables. The file is removed after reading. * ---------- */ static void -pgstat_read_db_statsfile(Oid databaseid, HTAB *tabhash, HTAB *funchash, - bool permanent) +pgstat_read_pgStatDBHashfile(PgStat_StatDBEntry *dbentry) { PgStat_StatTabEntry *tabentry; PgStat_StatTabEntry tabbuf; PgStat_StatFuncEntry funcbuf; PgStat_StatFuncEntry *funcentry; + dshash_table *tabhash =3D NULL; + dshash_table *funchash =3D NULL; FILE *fpin; int32 format_id; bool found; char statfile[MAXPGPATH]; = - get_dbstat_filename(permanent, false, databaseid, statfile, MAXPGPATH); + get_dbstat_filename(false, dbentry->databaseid, statfile, MAXPGPATH); = /* * Try to open the stats file. If it doesn't exist, the backends simply @@ -5360,7 +5593,7 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabha= sh, HTAB *funchash, if ((fpin =3D AllocateFile(statfile, PG_BINARY_R)) =3D=3D NULL) { if (errno !=3D ENOENT) - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errcode_for_file_access(), errmsg("could not open statistics file \"%s\": %m", statfile))); @@ -5373,14 +5606,14 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tab= hash, HTAB *funchash, if (fread(&format_id, 1, sizeof(format_id), fpin) !=3D sizeof(format_id)= || format_id !=3D PGSTAT_FILE_FORMAT_ID) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = /* - * We found an existing collector stats file. Read it and put all the - * hashtable entries into place. + * We found an existing statistics file. Read it and put all the hashtab= le + * entries into place. */ for (;;) { @@ -5393,31 +5626,35 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tab= hash, HTAB *funchash, if (fread(&tabbuf, 1, sizeof(PgStat_StatTabEntry), fpin) !=3D sizeof(PgStat_StatTabEntry)) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = - /* - * Skip if table data not wanted. - */ if (tabhash =3D=3D NULL) - break; + { + tabhash =3D dshash_create(area, &dsh_tblparams, 0); + dbentry->tables =3D + dshash_get_hash_table_handle(tabhash); + } = - tabentry =3D (PgStat_StatTabEntry *) hash_search(tabhash, - (void *) &tabbuf.tableid, - HASH_ENTER, &found); + tabentry =3D (PgStat_StatTabEntry *) + dshash_find_or_insert(tabhash, + (void *) &tabbuf.tableid, &found); = + /* don't allow duplicate entries */ if (found) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + dshash_release_lock(tabhash, tabentry); + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = memcpy(tabentry, &tabbuf, sizeof(tabbuf)); + dshash_release_lock(tabhash, tabentry); break; = /* @@ -5427,31 +5664,34 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tab= hash, HTAB *funchash, if (fread(&funcbuf, 1, sizeof(PgStat_StatFuncEntry), fpin) !=3D sizeof(PgStat_StatFuncEntry)) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = - /* - * Skip if function data not wanted. - */ if (funchash =3D=3D NULL) - break; + { + funchash =3D dshash_create(area, &dsh_tblparams, 0); + dbentry->functions =3D + dshash_get_hash_table_handle(funchash); + } = - funcentry =3D (PgStat_StatFuncEntry *) hash_search(funchash, - (void *) &funcbuf.functionid, - HASH_ENTER, &found); + funcentry =3D (PgStat_StatFuncEntry *) + dshash_find_or_insert(funchash, + (void *) &funcbuf.functionid, &found); = if (found) { - ereport(pgStatRunningInCollector ? LOG : WARNING, + dshash_release_lock(funchash, funcentry); + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; } = memcpy(funcentry, &funcbuf, sizeof(funcbuf)); + dshash_release_lock(funchash, funcentry); break; = /* @@ -5461,7 +5701,7 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *tabha= sh, HTAB *funchash, goto done; = default: - ereport(pgStatRunningInCollector ? LOG : WARNING, + ereport(LOG, (errmsg("corrupted statistics file \"%s\"", statfile))); goto done; @@ -5469,295 +5709,39 @@ pgstat_read_db_statsfile(Oid databaseid, HTAB *ta= bhash, HTAB *funchash, } = done: - FreeFile(fpin); - - if (permanent) - { - elog(DEBUG2, "removing permanent stats file \"%s\"", statfile); - unlink(statfile); - } -} - -/* ---------- - * pgstat_read_db_statsfile_timestamp() - - * - * Attempt to determine the timestamp of the last db statfile write. - * Returns true if successful; the timestamp is stored in *ts. - * - * This needs to be careful about handling databases for which no stats f= ile - * exists, such as databases without a stat entry or those not yet writte= n: - * - * - if there's a database entry in the global file, return the correspon= ding - * stats_timestamp value. - * - * - if there's no db stat entry (e.g. for a new or inactive database), - * there's no stats_timestamp value, but also nothing to write so we retu= rn - * the timestamp of the global statfile. - * ---------- - */ -static bool -pgstat_read_db_statsfile_timestamp(Oid databaseid, bool permanent, - TimestampTz *ts) -{ - PgStat_StatDBEntry dbentry; - PgStat_GlobalStats myGlobalStats; - PgStat_ArchiverStats myArchiverStats; - FILE *fpin; - int32 format_id; - const char *statfile =3D permanent ? PGSTAT_STAT_PERMANENT_FILENAME : pg= stat_stat_filename; - - /* - * Try to open the stats file. As above, anything but ENOENT is worthy = of - * complaining about. - */ - if ((fpin =3D AllocateFile(statfile, PG_BINARY_R)) =3D=3D NULL) - { - if (errno !=3D ENOENT) - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errcode_for_file_access(), - errmsg("could not open statistics file \"%s\": %m", - statfile))); - return false; - } - - /* - * Verify it's of the expected format. - */ - if (fread(&format_id, 1, sizeof(format_id), fpin) !=3D sizeof(format_id)= || - format_id !=3D PGSTAT_FILE_FORMAT_ID) - { - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", statfile))); - FreeFile(fpin); - return false; - } - - /* - * Read global stats struct - */ - if (fread(&myGlobalStats, 1, sizeof(myGlobalStats), - fpin) !=3D sizeof(myGlobalStats)) - { - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", statfile))); - FreeFile(fpin); - return false; - } - - /* - * Read archiver stats struct - */ - if (fread(&myArchiverStats, 1, sizeof(myArchiverStats), - fpin) !=3D sizeof(myArchiverStats)) - { - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", statfile))); - FreeFile(fpin); - return false; - } - - /* By default, we're going to return the timestamp of the global file. *= / - *ts =3D myGlobalStats.stats_timestamp; - - /* - * We found an existing collector stats file. Read it and look for a - * record for the requested database. If found, use its timestamp. - */ - for (;;) - { - switch (fgetc(fpin)) - { - /* - * 'D' A PgStat_StatDBEntry struct describing a database - * follows. - */ - case 'D': - if (fread(&dbentry, 1, offsetof(PgStat_StatDBEntry, tables), - fpin) !=3D offsetof(PgStat_StatDBEntry, tables)) - { - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", - statfile))); - goto done; - } - - /* - * If this is the DB we're looking for, save its timestamp and - * we're done. - */ - if (dbentry.databaseid =3D=3D databaseid) - { - *ts =3D dbentry.stats_timestamp; - goto done; - } - - break; - - case 'E': - goto done; - - default: - ereport(pgStatRunningInCollector ? LOG : WARNING, - (errmsg("corrupted statistics file \"%s\"", - statfile))); - goto done; - } - } + if (tabhash) + dshash_detach(tabhash); + if (funchash) + dshash_detach(funchash); = -done: FreeFile(fpin); - return true; -} - -/* - * If not already done, read the statistics collector stats file into - * some hash tables. The results will be kept until pgstat_clear_snapsho= t() - * is called (typically, at end of transaction). - */ -static void -backend_read_statsfile(void) -{ - TimestampTz min_ts =3D 0; - TimestampTz ref_ts =3D 0; - Oid inquiry_db; - int count; - - /* already read it? */ - if (pgStatDBHash) - return; - Assert(!pgStatRunningInCollector); - - /* - * In a normal backend, we check staleness of the data for our own DB, a= nd - * so we send MyDatabaseId in inquiry messages. In the autovac launcher= , - * check staleness of the shared-catalog data, and send InvalidOid in - * inquiry messages so as not to force writing unnecessary data. - */ - if (IsAutoVacuumLauncherProcess()) - inquiry_db =3D InvalidOid; - else - inquiry_db =3D MyDatabaseId; - - /* - * Loop until fresh enough stats file is available or we ran out of time= . - * The stats inquiry message is sent repeatedly in case collector drops - * it; but not every single time, as that just swamps the collector. - */ - for (count =3D 0; count < PGSTAT_POLL_LOOP_COUNT; count++) - { - bool ok; - TimestampTz file_ts =3D 0; - TimestampTz cur_ts; - - CHECK_FOR_INTERRUPTS(); - - ok =3D pgstat_read_db_statsfile_timestamp(inquiry_db, false, &file_ts); = - cur_ts =3D GetCurrentTimestamp(); - /* Calculate min acceptable timestamp, if we didn't already */ - if (count =3D=3D 0 || cur_ts < ref_ts) - { - /* - * We set the minimum acceptable timestamp to PGSTAT_STAT_INTERVAL - * msec before now. This indirectly ensures that the collector - * needn't write the file more often than PGSTAT_STAT_INTERVAL. In - * an autovacuum worker, however, we want a lower delay to avoid - * using stale data, so we use PGSTAT_RETRY_DELAY (since the - * number of workers is low, this shouldn't be a problem). - * - * We don't recompute min_ts after sleeping, except in the - * unlikely case that cur_ts went backwards. So we might end up - * accepting a file a bit older than PGSTAT_STAT_INTERVAL. In - * practice that shouldn't happen, though, as long as the sleep - * time is less than PGSTAT_STAT_INTERVAL; and we don't want to - * tell the collector that our cutoff time is less than what we'd - * actually accept. - */ - ref_ts =3D cur_ts; - if (IsAutoVacuumWorkerProcess()) - min_ts =3D TimestampTzPlusMilliseconds(ref_ts, - -PGSTAT_RETRY_DELAY); - else - min_ts =3D TimestampTzPlusMilliseconds(ref_ts, - -PGSTAT_STAT_INTERVAL); - } - - /* - * If the file timestamp is actually newer than cur_ts, we must have - * had a clock glitch (system time went backwards) or there is clock - * skew between our processor and the stats collector's processor. - * Accept the file, but send an inquiry message anyway to make - * pgstat_recv_inquiry do a sanity check on the collector's time. - */ - if (ok && file_ts > cur_ts) - { - /* - * A small amount of clock skew between processors isn't terribly - * surprising, but a large difference is worth logging. We - * arbitrarily define "large" as 1000 msec. - */ - if (file_ts >=3D TimestampTzPlusMilliseconds(cur_ts, 1000)) - { - char *filetime; - char *mytime; - - /* Copy because timestamptz_to_str returns a static buffer */ - filetime =3D pstrdup(timestamptz_to_str(file_ts)); - mytime =3D pstrdup(timestamptz_to_str(cur_ts)); - elog(LOG, "stats collector's time %s is later than backend local time= %s", - filetime, mytime); - pfree(filetime); - pfree(mytime); - } - - pgstat_send_inquiry(cur_ts, min_ts, inquiry_db); - break; - } - - /* Normal acceptance case: file is not older than cutoff time */ - if (ok && file_ts >=3D min_ts) - break; - - /* Not there or too old, so kick the collector and wait a bit */ - if ((count % PGSTAT_INQ_LOOP_COUNT) =3D=3D 0) - pgstat_send_inquiry(cur_ts, min_ts, inquiry_db); - - pg_usleep(PGSTAT_RETRY_DELAY * 1000L); - } - - if (count >=3D PGSTAT_POLL_LOOP_COUNT) - ereport(LOG, - (errmsg("using stale statistics instead of current ones " - "because stats collector is not responding"))); - - /* - * Autovacuum launcher wants stats about all databases, but a shallow re= ad - * is sufficient. Regular backends want a deep read for just the tables - * they can see (MyDatabaseId + shared catalogs). - */ - if (IsAutoVacuumLauncherProcess()) - pgStatDBHash =3D pgstat_read_statsfiles(InvalidOid, false, false); - else - pgStatDBHash =3D pgstat_read_statsfiles(MyDatabaseId, false, true); + elog(DEBUG2, "removing permanent stats file \"%s\"", statfile); + unlink(statfile); } = - /* ---------- * pgstat_setup_memcxt() - * - * Create pgStatLocalContext, if not already done. + * Create pgStatLocalContext and pgStatSnapshotContext, if not already do= ne. * ---------- */ static void pgstat_setup_memcxt(void) { if (!pgStatLocalContext) - pgStatLocalContext =3D AllocSetContextCreate(TopMemoryContext, - "Statistics snapshot", - ALLOCSET_SMALL_SIZES); + pgStatLocalContext =3D + AllocSetContextCreate(TopMemoryContext, + "Backend statistics snapshot", + ALLOCSET_SMALL_SIZES); + + if (!pgStatSnapshotContext) + pgStatSnapshotContext =3D + AllocSetContextCreate(TopMemoryContext, + "Database statistics snapshot", + ALLOCSET_SMALL_SIZES); } = - /* ---------- * pgstat_clear_snapshot() - * @@ -5773,739 +5757,223 @@ pgstat_clear_snapshot(void) { /* Release memory, if any was allocated */ if (pgStatLocalContext) + { MemoryContextDelete(pgStatLocalContext); = - /* Reset variables */ - pgStatLocalContext =3D NULL; - pgStatDBHash =3D NULL; - localBackendStatusTable =3D NULL; - localNumBackends =3D 0; -} + /* Reset variables */ + pgStatLocalContext =3D NULL; + localBackendStatusTable =3D NULL; + localNumBackends =3D 0; + } = + if (pgStatSnapshotContext) + clear_snapshot =3D true; +} = -/* ---------- - * pgstat_recv_inquiry() - - * - * Process stat inquiry requests. - * ---------- - */ -static void -pgstat_recv_inquiry(PgStat_MsgInquiry *msg, int len) +static bool +pgstat_update_tabentry(dshash_table *tabhash, PgStat_TableStatus *stat, + bool nowait) { - PgStat_StatDBEntry *dbentry; + PgStat_StatTabEntry *tabentry; + bool found; = - elog(DEBUG2, "received inquiry for database %u", msg->databaseid); + if (tabhash =3D=3D NULL) + return false; = - /* - * If there's already a write request for this DB, there's nothing to do= . - * - * Note that if a request is found, we return early and skip the below - * check for clock skew. This is okay, since the only way for a DB - * request to be present in the list is that we have been here since the - * last write round. It seems sufficient to check for clock skew once p= er - * write round. - */ - if (list_member_oid(pending_write_requests, msg->databaseid)) - return; + tabentry =3D (PgStat_StatTabEntry *) + dshash_find_or_insert_extended(tabhash, (void *) &(stat->t_id), + &found, nowait); = - /* - * Check to see if we last wrote this database at a time >=3D the reques= ted - * cutoff time. If so, this is a stale request that was generated befor= e - * we updated the DB file, and we don't need to do so again. - * - * If the requestor's local clock time is older than stats_timestamp, we - * should suspect a clock glitch, ie system time going backwards; though - * the more likely explanation is just delayed message receipt. It is - * worth expending a GetCurrentTimestamp call to be sure, since a large - * retreat in the system clock reading could otherwise cause us to negle= ct - * to update the stats file for a long time. - */ - dbentry =3D pgstat_get_db_entry(msg->databaseid, false); - if (dbentry =3D=3D NULL) + /* failed to acquire lock */ + if (tabentry =3D=3D NULL) + return false; + + if (!found) { /* - * We have no data for this DB. Enter a write request anyway so that - * the global stats will get updated. This is needed to prevent - * backend_read_statsfile from waiting for data that we cannot supply, - * in the case of a new DB that nobody has yet reported any stats for. - * See the behavior of pgstat_read_db_statsfile_timestamp. + * If it's a new table entry, initialize counters to the values we + * just got. */ + tabentry->numscans =3D stat->t_counts.t_numscans; + tabentry->tuples_returned =3D stat->t_counts.t_tuples_returned; + tabentry->tuples_fetched =3D stat->t_counts.t_tuples_fetched; + tabentry->tuples_inserted =3D stat->t_counts.t_tuples_inserted; + tabentry->tuples_updated =3D stat->t_counts.t_tuples_updated; + tabentry->tuples_deleted =3D stat->t_counts.t_tuples_deleted; + tabentry->tuples_hot_updated =3D stat->t_counts.t_tuples_hot_updated; + tabentry->n_live_tuples =3D stat->t_counts.t_delta_live_tuples; + tabentry->n_dead_tuples =3D stat->t_counts.t_delta_dead_tuples; + tabentry->changes_since_analyze =3D stat->t_counts.t_changed_tuples; + tabentry->blocks_fetched =3D stat->t_counts.t_blocks_fetched; + tabentry->blocks_hit =3D stat->t_counts.t_blocks_hit; + + tabentry->vacuum_timestamp =3D 0; + tabentry->vacuum_count =3D 0; + tabentry->autovac_vacuum_timestamp =3D 0; + tabentry->autovac_vacuum_count =3D 0; + tabentry->analyze_timestamp =3D 0; + tabentry->analyze_count =3D 0; + tabentry->autovac_analyze_timestamp =3D 0; + tabentry->autovac_analyze_count =3D 0; } - else if (msg->clock_time < dbentry->stats_timestamp) + else { - TimestampTz cur_ts =3D GetCurrentTimestamp(); - - if (cur_ts < dbentry->stats_timestamp) - { - /* - * Sure enough, time went backwards. Force a new stats file write - * to get back in sync; but first, log a complaint. - */ - char *writetime; - char *mytime; - - /* Copy because timestamptz_to_str returns a static buffer */ - writetime =3D pstrdup(timestamptz_to_str(dbentry->stats_timestamp)); - mytime =3D pstrdup(timestamptz_to_str(cur_ts)); - elog(LOG, - "stats_timestamp %s is later than collector's time %s for database %= u", - writetime, mytime, dbentry->databaseid); - pfree(writetime); - pfree(mytime); - } - else + /* + * Otherwise add the values to the existing entry. + */ + tabentry->numscans +=3D stat->t_counts.t_numscans; + tabentry->tuples_returned +=3D stat->t_counts.t_tuples_returned; + tabentry->tuples_fetched +=3D stat->t_counts.t_tuples_fetched; + tabentry->tuples_inserted +=3D stat->t_counts.t_tuples_inserted; + tabentry->tuples_updated +=3D stat->t_counts.t_tuples_updated; + tabentry->tuples_deleted +=3D stat->t_counts.t_tuples_deleted; + tabentry->tuples_hot_updated +=3D stat->t_counts.t_tuples_hot_updated; + /* If table was truncated, first reset the live/dead counters */ + if (stat->t_counts.t_truncated) { - /* - * Nope, it's just an old request. Assuming msg's clock_time is - * >=3D its cutoff_time, it must be stale, so we can ignore it. - */ - return; + tabentry->n_live_tuples =3D 0; + tabentry->n_dead_tuples =3D 0; } - } - else if (msg->cutoff_time <=3D dbentry->stats_timestamp) - { - /* Stale request, ignore it */ - return; + tabentry->n_live_tuples +=3D stat->t_counts.t_delta_live_tuples; + tabentry->n_dead_tuples +=3D stat->t_counts.t_delta_dead_tuples; + tabentry->changes_since_analyze +=3D stat->t_counts.t_changed_tuples; + tabentry->blocks_fetched +=3D stat->t_counts.t_blocks_fetched; + tabentry->blocks_hit +=3D stat->t_counts.t_blocks_hit; } = - /* - * We need to write this DB, so create a request. - */ - pending_write_requests =3D lappend_oid(pending_write_requests, - msg->databaseid); -} + /* Clamp n_live_tuples in case of negative delta_live_tuples */ + tabentry->n_live_tuples =3D Max(tabentry->n_live_tuples, 0); + /* Likewise for n_dead_tuples */ + tabentry->n_dead_tuples =3D Max(tabentry->n_dead_tuples, 0); = + dshash_release_lock(tabhash, tabentry); + + return true; +} = -/* ---------- - * pgstat_recv_tabstat() - - * - * Count what the backend has done. - * ---------- - */ static void -pgstat_recv_tabstat(PgStat_MsgTabstat *msg, int len) +pgstat_update_dbentry(PgStat_StatDBEntry *dbentry, PgStat_TableStatus *st= at) { - PgStat_StatDBEntry *dbentry; - PgStat_StatTabEntry *tabentry; - int i; - bool found; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - /* - * Update database-wide stats. + * Add per-table stats to the per-database entry, too. */ - dbentry->n_xact_commit +=3D (PgStat_Counter) (msg->m_xact_commit); - dbentry->n_xact_rollback +=3D (PgStat_Counter) (msg->m_xact_rollback); - dbentry->n_block_read_time +=3D msg->m_block_read_time; - dbentry->n_block_write_time +=3D msg->m_block_write_time; + LWLockAcquire(&dbentry->lock, LW_EXCLUSIVE); + dbentry->n_tuples_returned +=3D stat->t_counts.t_tuples_returned; + dbentry->n_tuples_fetched +=3D stat->t_counts.t_tuples_fetched; + dbentry->n_tuples_inserted +=3D stat->t_counts.t_tuples_inserted; + dbentry->n_tuples_updated +=3D stat->t_counts.t_tuples_updated; + dbentry->n_tuples_deleted +=3D stat->t_counts.t_tuples_deleted; + dbentry->n_blocks_fetched +=3D stat->t_counts.t_blocks_fetched; + dbentry->n_blocks_hit +=3D stat->t_counts.t_blocks_hit; + LWLockRelease(&dbentry->lock); +} = - /* - * Process all table entries in the message. - */ - for (i =3D 0; i < msg->m_nentries; i++) - { - PgStat_TableEntry *tabmsg =3D &(msg->m_entry[i]); +/* + * Lookup shared stats hash table for the specified database. Returns NUL= L + * when PGSTAT_NOWAIT and required lock cannot be acquired. + */ +static PgStat_StatDBEntry * +pgstat_get_db_entry(Oid databaseid, int op, PgStat_TableLookupResult *sta= tus) +{ + PgStat_StatDBEntry *result; + bool nowait =3D ((op & PGSTAT_NOWAIT) !=3D 0); + bool lock_acquired =3D true; + bool found =3D true; = - tabentry =3D (PgStat_StatTabEntry *) hash_search(dbentry->tables, - (void *) &(tabmsg->t_id), - HASH_ENTER, &found); + if (!IsUnderPostmaster || !pgStatDBHash) + return NULL; = - if (!found) - { - /* - * If it's a new table entry, initialize counters to the values we - * just got. - */ - tabentry->numscans =3D tabmsg->t_counts.t_numscans; - tabentry->tuples_returned =3D tabmsg->t_counts.t_tuples_returned; - tabentry->tuples_fetched =3D tabmsg->t_counts.t_tuples_fetched; - tabentry->tuples_inserted =3D tabmsg->t_counts.t_tuples_inserted; - tabentry->tuples_updated =3D tabmsg->t_counts.t_tuples_updated; - tabentry->tuples_deleted =3D tabmsg->t_counts.t_tuples_deleted; - tabentry->tuples_hot_updated =3D tabmsg->t_counts.t_tuples_hot_updated= ; - tabentry->n_live_tuples =3D tabmsg->t_counts.t_delta_live_tuples; - tabentry->n_dead_tuples =3D tabmsg->t_counts.t_delta_dead_tuples; - tabentry->changes_since_analyze =3D tabmsg->t_counts.t_changed_tuples; - tabentry->blocks_fetched =3D tabmsg->t_counts.t_blocks_fetched; - tabentry->blocks_hit =3D tabmsg->t_counts.t_blocks_hit; - - tabentry->vacuum_timestamp =3D 0; - tabentry->vacuum_count =3D 0; - tabentry->autovac_vacuum_timestamp =3D 0; - tabentry->autovac_vacuum_count =3D 0; - tabentry->analyze_timestamp =3D 0; - tabentry->analyze_count =3D 0; - tabentry->autovac_analyze_timestamp =3D 0; - tabentry->autovac_analyze_count =3D 0; - } - else + /* Lookup or create the hash table entry for this database */ + if (op & PGSTAT_EXCLUSIVE) + { + result =3D (PgStat_StatDBEntry *) + dshash_find_or_insert_extended(pgStatDBHash, &databaseid, + &found, nowait); + if (result =3D=3D NULL) + lock_acquired =3D false; + else if (!found) { /* - * Otherwise add the values to the existing entry. + * If not found, initialize the new one. This creates empty hash + * tables hash, too. */ - tabentry->numscans +=3D tabmsg->t_counts.t_numscans; - tabentry->tuples_returned +=3D tabmsg->t_counts.t_tuples_returned; - tabentry->tuples_fetched +=3D tabmsg->t_counts.t_tuples_fetched; - tabentry->tuples_inserted +=3D tabmsg->t_counts.t_tuples_inserted; - tabentry->tuples_updated +=3D tabmsg->t_counts.t_tuples_updated; - tabentry->tuples_deleted +=3D tabmsg->t_counts.t_tuples_deleted; - tabentry->tuples_hot_updated +=3D tabmsg->t_counts.t_tuples_hot_update= d; - /* If table was truncated, first reset the live/dead counters */ - if (tabmsg->t_counts.t_truncated) - { - tabentry->n_live_tuples =3D 0; - tabentry->n_dead_tuples =3D 0; - } - tabentry->n_live_tuples +=3D tabmsg->t_counts.t_delta_live_tuples; - tabentry->n_dead_tuples +=3D tabmsg->t_counts.t_delta_dead_tuples; - tabentry->changes_since_analyze +=3D tabmsg->t_counts.t_changed_tuples= ; - tabentry->blocks_fetched +=3D tabmsg->t_counts.t_blocks_fetched; - tabentry->blocks_hit +=3D tabmsg->t_counts.t_blocks_hit; + init_dbentry(result); + reset_dbentry_counters(result); } - - /* Clamp n_live_tuples in case of negative delta_live_tuples */ - tabentry->n_live_tuples =3D Max(tabentry->n_live_tuples, 0); - /* Likewise for n_dead_tuples */ - tabentry->n_dead_tuples =3D Max(tabentry->n_dead_tuples, 0); - - /* - * Add per-table stats to the per-database entry, too. - */ - dbentry->n_tuples_returned +=3D tabmsg->t_counts.t_tuples_returned; - dbentry->n_tuples_fetched +=3D tabmsg->t_counts.t_tuples_fetched; - dbentry->n_tuples_inserted +=3D tabmsg->t_counts.t_tuples_inserted; - dbentry->n_tuples_updated +=3D tabmsg->t_counts.t_tuples_updated; - dbentry->n_tuples_deleted +=3D tabmsg->t_counts.t_tuples_deleted; - dbentry->n_blocks_fetched +=3D tabmsg->t_counts.t_blocks_fetched; - dbentry->n_blocks_hit +=3D tabmsg->t_counts.t_blocks_hit; - } -} - - -/* ---------- - * pgstat_recv_tabpurge() - - * - * Arrange for dead table removal. - * ---------- - */ -static void -pgstat_recv_tabpurge(PgStat_MsgTabpurge *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - int i; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, false); - - /* - * No need to purge if we don't even know the database. - */ - if (!dbentry || !dbentry->tables) - return; - - /* - * Process all table entries in the message. - */ - for (i =3D 0; i < msg->m_nentries; i++) - { - /* Remove from hashtable if present; we don't care if it's not. */ - (void) hash_search(dbentry->tables, - (void *) &(msg->m_tableid[i]), - HASH_REMOVE, NULL); - } -} - - -/* ---------- - * pgstat_recv_dropdb() - - * - * Arrange for dead database removal - * ---------- - */ -static void -pgstat_recv_dropdb(PgStat_MsgDropdb *msg, int len) -{ - Oid dbid =3D msg->m_databaseid; - PgStat_StatDBEntry *dbentry; - - /* - * Lookup the database in the hashtable. - */ - dbentry =3D pgstat_get_db_entry(dbid, false); - - /* - * If found, remove it (along with the db statfile). - */ - if (dbentry) - { - char statfile[MAXPGPATH]; - - get_dbstat_filename(false, false, dbid, statfile, MAXPGPATH); - - elog(DEBUG2, "removing stats file \"%s\"", statfile); - unlink(statfile); - - if (dbentry->tables !=3D NULL) - hash_destroy(dbentry->tables); - if (dbentry->functions !=3D NULL) - hash_destroy(dbentry->functions); - - if (hash_search(pgStatDBHash, - (void *) &dbid, - HASH_REMOVE, NULL) =3D=3D NULL) - ereport(ERROR, - (errmsg("database hash table corrupted during cleanup --- abort"))); - } -} - - -/* ---------- - * pgstat_recv_resetcounter() - - * - * Reset the statistics for the specified database. - * ---------- - */ -static void -pgstat_recv_resetcounter(PgStat_MsgResetcounter *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - /* - * Lookup the database in the hashtable. Nothing to do if not there. - */ - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, false); - - if (!dbentry) - return; - - /* - * We simply throw away all the database's table entries by recreating a - * new hash table for them. - */ - if (dbentry->tables !=3D NULL) - hash_destroy(dbentry->tables); - if (dbentry->functions !=3D NULL) - hash_destroy(dbentry->functions); - - dbentry->tables =3D NULL; - dbentry->functions =3D NULL; - - /* - * Reset database-level stats, too. This creates empty hash tables for - * tables and functions. - */ - reset_dbentry_counters(dbentry); -} - -/* ---------- - * pgstat_recv_resetsharedcounter() - - * - * Reset some shared statistics of the cluster. - * ---------- - */ -static void -pgstat_recv_resetsharedcounter(PgStat_MsgResetsharedcounter *msg, int len= ) -{ - if (msg->m_resettarget =3D=3D RESET_BGWRITER) - { - /* Reset the global background writer statistics for the cluster. */ - memset(&globalStats, 0, sizeof(globalStats)); - globalStats.stat_reset_timestamp =3D GetCurrentTimestamp(); - } - else if (msg->m_resettarget =3D=3D RESET_ARCHIVER) - { - /* Reset the archiver statistics for the cluster. */ - memset(&archiverStats, 0, sizeof(archiverStats)); - archiverStats.stat_reset_timestamp =3D GetCurrentTimestamp(); - } - - /* - * Presumably the sender of this message validated the target, don't - * complain here if it's not valid - */ -} - -/* ---------- - * pgstat_recv_resetsinglecounter() - - * - * Reset a statistics for a single object - * ---------- - */ -static void -pgstat_recv_resetsinglecounter(PgStat_MsgResetsinglecounter *msg, int len= ) -{ - PgStat_StatDBEntry *dbentry; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, false); - - if (!dbentry) - return; - - /* Set the reset timestamp for the whole database */ - dbentry->stat_reset_timestamp =3D GetCurrentTimestamp(); - - /* Remove object if it exists, ignore it if not */ - if (msg->m_resettype =3D=3D RESET_TABLE) - (void) hash_search(dbentry->tables, (void *) &(msg->m_objectid), - HASH_REMOVE, NULL); - else if (msg->m_resettype =3D=3D RESET_FUNCTION) - (void) hash_search(dbentry->functions, (void *) &(msg->m_objectid), - HASH_REMOVE, NULL); -} - -/* ---------- - * pgstat_recv_autovac() - - * - * Process an autovacuum signalling message. - * ---------- - */ -static void -pgstat_recv_autovac(PgStat_MsgAutovacStart *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - /* - * Store the last autovacuum time in the database's hashtable entry. - */ - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - dbentry->last_autovac_time =3D msg->m_start_time; -} - -/* ---------- - * pgstat_recv_vacuum() - - * - * Process a VACUUM message. - * ---------- - */ -static void -pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - PgStat_StatTabEntry *tabentry; - - /* - * Store the data in the table's hashtable entry. - */ - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - tabentry =3D pgstat_get_tab_entry(dbentry, msg->m_tableoid, true); - - tabentry->n_live_tuples =3D msg->m_live_tuples; - tabentry->n_dead_tuples =3D msg->m_dead_tuples; - - if (msg->m_autovacuum) - { - tabentry->autovac_vacuum_timestamp =3D msg->m_vacuumtime; - tabentry->autovac_vacuum_count++; - } - else - { - tabentry->vacuum_timestamp =3D msg->m_vacuumtime; - tabentry->vacuum_count++; - } -} - -/* ---------- - * pgstat_recv_analyze() - - * - * Process an ANALYZE message. - * ---------- - */ -static void -pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - PgStat_StatTabEntry *tabentry; - - /* - * Store the data in the table's hashtable entry. - */ - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - tabentry =3D pgstat_get_tab_entry(dbentry, msg->m_tableoid, true); - - tabentry->n_live_tuples =3D msg->m_live_tuples; - tabentry->n_dead_tuples =3D msg->m_dead_tuples; - - /* - * If commanded, reset changes_since_analyze to zero. This forgets any - * changes that were committed while the ANALYZE was in progress, but we - * have no good way to estimate how many of those there were. - */ - if (msg->m_resetcounter) - tabentry->changes_since_analyze =3D 0; - - if (msg->m_autovacuum) - { - tabentry->autovac_analyze_timestamp =3D msg->m_analyzetime; - tabentry->autovac_analyze_count++; - } - else - { - tabentry->analyze_timestamp =3D msg->m_analyzetime; - tabentry->analyze_count++; - } -} - - -/* ---------- - * pgstat_recv_archiver() - - * - * Process a ARCHIVER message. - * ---------- - */ -static void -pgstat_recv_archiver(PgStat_MsgArchiver *msg, int len) -{ - if (msg->m_failed) - { - /* Failed archival attempt */ - ++archiverStats.failed_count; - memcpy(archiverStats.last_failed_wal, msg->m_xlog, - sizeof(archiverStats.last_failed_wal)); - archiverStats.last_failed_timestamp =3D msg->m_timestamp; } else { - /* Successful archival operation */ - ++archiverStats.archived_count; - memcpy(archiverStats.last_archived_wal, msg->m_xlog, - sizeof(archiverStats.last_archived_wal)); - archiverStats.last_archived_timestamp =3D msg->m_timestamp; - } -} - -/* ---------- - * pgstat_recv_bgwriter() - - * - * Process a BGWRITER message. - * ---------- - */ -static void -pgstat_recv_bgwriter(PgStat_MsgBgWriter *msg, int len) -{ - globalStats.timed_checkpoints +=3D msg->m_timed_checkpoints; - globalStats.requested_checkpoints +=3D msg->m_requested_checkpoints; - globalStats.checkpoint_write_time +=3D msg->m_checkpoint_write_time; - globalStats.checkpoint_sync_time +=3D msg->m_checkpoint_sync_time; - globalStats.buf_written_checkpoints +=3D msg->m_buf_written_checkpoints; - globalStats.buf_written_clean +=3D msg->m_buf_written_clean; - globalStats.maxwritten_clean +=3D msg->m_maxwritten_clean; - globalStats.buf_written_backend +=3D msg->m_buf_written_backend; - globalStats.buf_fsync_backend +=3D msg->m_buf_fsync_backend; - globalStats.buf_alloc +=3D msg->m_buf_alloc; -} - -/* ---------- - * pgstat_recv_recoveryconflict() - - * - * Process a RECOVERYCONFLICT message. - * ---------- - */ -static void -pgstat_recv_recoveryconflict(PgStat_MsgRecoveryConflict *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - switch (msg->m_reason) - { - case PROCSIG_RECOVERY_CONFLICT_DATABASE: - - /* - * Since we drop the information about the database as soon as it - * replicates, there is no point in counting these conflicts. - */ - break; - case PROCSIG_RECOVERY_CONFLICT_TABLESPACE: - dbentry->n_conflict_tablespace++; - break; - case PROCSIG_RECOVERY_CONFLICT_LOCK: - dbentry->n_conflict_lock++; - break; - case PROCSIG_RECOVERY_CONFLICT_SNAPSHOT: - dbentry->n_conflict_snapshot++; - break; - case PROCSIG_RECOVERY_CONFLICT_BUFFERPIN: - dbentry->n_conflict_bufferpin++; - break; - case PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK: - dbentry->n_conflict_startup_deadlock++; - break; + result =3D (PgStat_StatDBEntry *) + dshash_find_extended(pgStatDBHash, &databaseid, true, nowait, + nowait ? &lock_acquired : NULL); + if (result =3D=3D NULL) + found =3D false; } -} - -/* ---------- - * pgstat_recv_deadlock() - - * - * Process a DEADLOCK message. - * ---------- - */ -static void -pgstat_recv_deadlock(PgStat_MsgDeadlock *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - dbentry->n_deadlocks++; -} - -/* ---------- - * pgstat_recv_checksum_failure() - - * - * Process a CHECKSUMFAILURE message. - * ---------- - */ -static void -pgstat_recv_checksum_failure(PgStat_MsgChecksumFailure *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - dbentry->n_checksum_failures +=3D msg->m_failurecount; - dbentry->last_checksum_failure =3D msg->m_failure_time; -} - -/* ---------- - * pgstat_recv_tempfile() - - * - * Process a TEMPFILE message. - * ---------- - */ -static void -pgstat_recv_tempfile(PgStat_MsgTempFile *msg, int len) -{ - PgStat_StatDBEntry *dbentry; - - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - dbentry->n_temp_bytes +=3D msg->m_filesize; - dbentry->n_temp_files +=3D 1; -} - -/* ---------- - * pgstat_recv_funcstat() - - * - * Count what the backend has done. - * ---------- - */ -static void -pgstat_recv_funcstat(PgStat_MsgFuncstat *msg, int len) -{ - PgStat_FunctionEntry *funcmsg =3D &(msg->m_entry[0]); - PgStat_StatDBEntry *dbentry; - PgStat_StatFuncEntry *funcentry; - int i; - bool found; = - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, true); - - /* - * Process all function entries in the message. - */ - for (i =3D 0; i < msg->m_nentries; i++, funcmsg++) + /* Set return status if requested */ + if (status) { - funcentry =3D (PgStat_StatFuncEntry *) hash_search(dbentry->functions, - (void *) &(funcmsg->f_id), - HASH_ENTER, &found); - - if (!found) + if (!lock_acquired) { - /* - * If it's a new function entry, initialize counters to the values - * we just got. - */ - funcentry->f_numcalls =3D funcmsg->f_numcalls; - funcentry->f_total_time =3D funcmsg->f_total_time; - funcentry->f_self_time =3D funcmsg->f_self_time; + Assert(nowait); + *status =3D LOCK_FAILED; } + else if (!found) + *status =3D NOT_FOUND; else - { - /* - * Otherwise add the values to the existing entry. - */ - funcentry->f_numcalls +=3D funcmsg->f_numcalls; - funcentry->f_total_time +=3D funcmsg->f_total_time; - funcentry->f_self_time +=3D funcmsg->f_self_time; - } + *status =3D FOUND; } + + return result; } = -/* ---------- - * pgstat_recv_funcpurge() - - * - * Arrange for dead function removal. - * ---------- +/* + * Lookup the hash table entry for the specified table. If no hash + * table entry exists, initialize it, if the create parameter is true. + * Else, return NULL. */ -static void -pgstat_recv_funcpurge(PgStat_MsgFuncpurge *msg, int len) +static PgStat_StatTabEntry * +pgstat_get_tab_entry(dshash_table *table, Oid tableoid, bool create) { - PgStat_StatDBEntry *dbentry; - int i; + PgStat_StatTabEntry *result; + bool found; = - dbentry =3D pgstat_get_db_entry(msg->m_databaseid, false); + /* Lookup or create the hash table entry for this table */ + if (create) + result =3D (PgStat_StatTabEntry *) + dshash_find_or_insert(table, &tableoid, &found); + else + result =3D (PgStat_StatTabEntry *) dshash_find(table, &tableoid, false)= ; = - /* - * No need to purge if we don't even know the database. - */ - if (!dbentry || !dbentry->functions) - return; + if (!create && !found) + return NULL; = - /* - * Process all function entries in the message. - */ - for (i =3D 0; i < msg->m_nentries; i++) + /* If not found, initialize the new one. */ + if (!found) { - /* Remove from hashtable if present; we don't care if it's not. */ - (void) hash_search(dbentry->functions, - (void *) &(msg->m_functionid[i]), - HASH_REMOVE, NULL); + result->numscans =3D 0; + result->tuples_returned =3D 0; + result->tuples_fetched =3D 0; + result->tuples_inserted =3D 0; + result->tuples_updated =3D 0; + result->tuples_deleted =3D 0; + result->tuples_hot_updated =3D 0; + result->n_live_tuples =3D 0; + result->n_dead_tuples =3D 0; + result->changes_since_analyze =3D 0; + result->blocks_fetched =3D 0; + result->blocks_hit =3D 0; + result->vacuum_timestamp =3D 0; + result->vacuum_count =3D 0; + result->autovac_vacuum_timestamp =3D 0; + result->autovac_vacuum_count =3D 0; + result->analyze_timestamp =3D 0; + result->analyze_count =3D 0; + result->autovac_analyze_timestamp =3D 0; + result->autovac_analyze_count =3D 0; } -} = -/* ---------- - * pgstat_write_statsfile_needed() - - * - * Do we need to write out any stats files? - * ---------- - */ -static bool -pgstat_write_statsfile_needed(void) -{ - if (pending_write_requests !=3D NIL) - return true; - - /* Everything was written recently */ - return false; -} - -/* ---------- - * pgstat_db_requested() - - * - * Checks whether stats for a particular DB need to be written to a file. - * ---------- - */ -static bool -pgstat_db_requested(Oid databaseid) -{ - /* - * If any requests are outstanding at all, we should write the stats for - * shared catalogs (the "database" with OID 0). This ensures that - * backends will see up-to-date stats for shared catalogs, even though - * they send inquiry messages mentioning only their own DB. - */ - if (databaseid =3D=3D InvalidOid && pending_write_requests !=3D NIL) - return true; - - /* Search to see if there's an open request to write this database. */ - if (list_member_oid(pending_write_requests, databaseid)) - return true; - - return false; + return result; } = /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/= postmaster.c index 5df9333..59bf597 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -255,7 +255,6 @@ static pid_t StartupPID =3D 0, WalReceiverPID =3D 0, AutoVacPID =3D 0, PgArchPID =3D 0, - PgStatPID =3D 0, SysLoggerPID =3D 0; = /* Startup process's status */ @@ -503,7 +502,6 @@ typedef struct PGPROC *AuxiliaryProcs; PGPROC *PreparedXactProcs; PMSignalData *PMSignalState; - InheritableSocket pgStatSock; pid_t PostmasterPid; TimestampTz PgStartTime; TimestampTz PgReloadTime; @@ -1327,12 +1325,6 @@ PostmasterMain(int argc, char *argv[]) RemovePgTempFiles(); = /* - * Initialize stats collection subsystem (this does NOT start the - * collector process!) - */ - pgstat_init(); - - /* * Initialize the autovacuum subsystem (again, no process start yet) */ autovac_init(); @@ -1780,11 +1772,6 @@ ServerLoop(void) start_autovac_launcher =3D false; /* signal processed */ } = - /* If we have lost the stats collector, try to start a new one */ - if (PgStatPID =3D=3D 0 && - (pmState =3D=3D PM_RUN || pmState =3D=3D PM_HOT_STANDBY)) - PgStatPID =3D pgstat_start(); - /* If we have lost the archiver, try to start a new one. */ if (PgArchPID =3D=3D 0 && PgArchStartupAllowed()) PgArchPID =3D StartArchiver(); @@ -2689,8 +2676,6 @@ SIGHUP_handler(SIGNAL_ARGS) signal_child(PgArchPID, SIGHUP); if (SysLoggerPID !=3D 0) signal_child(SysLoggerPID, SIGHUP); - if (PgStatPID !=3D 0) - signal_child(PgStatPID, SIGHUP); = /* Reload authentication config files too */ if (!load_hba()) @@ -3053,8 +3038,6 @@ reaper(SIGNAL_ARGS) AutoVacPID =3D StartAutoVacLauncher(); if (PgArchStartupAllowed() && PgArchPID =3D=3D 0) PgArchPID =3D StartArchiver(); - if (PgStatPID =3D=3D 0) - PgStatPID =3D pgstat_start(); = /* workers may be scheduled to start now */ maybe_start_bgworkers(); @@ -3121,13 +3104,6 @@ reaper(SIGNAL_ARGS) SignalChildren(SIGUSR2); = pmState =3D PM_SHUTDOWN_2; - - /* - * We can also shut down the stats collector now; there's - * nothing left for it to do. - */ - if (PgStatPID !=3D 0) - signal_child(PgStatPID, SIGQUIT); } else { @@ -3200,22 +3176,6 @@ reaper(SIGNAL_ARGS) continue; } = - /* - * Was it the statistics collector? If so, just try to start a new - * one; no need to force reset of the rest of the system. (If fail, - * we'll try again in future cycles of the main loop.) - */ - if (pid =3D=3D PgStatPID) - { - PgStatPID =3D 0; - if (!EXIT_STATUS_0(exitstatus)) - LogChildExit(LOG, _("statistics collector process"), - pid, exitstatus); - if (pmState =3D=3D PM_RUN || pmState =3D=3D PM_HOT_STANDBY) - PgStatPID =3D pgstat_start(); - continue; - } - /* Was it the system logger? If so, try to start a new one */ if (pid =3D=3D SysLoggerPID) { @@ -3676,22 +3636,6 @@ HandleChildCrash(int pid, int exitstatus, const cha= r *procname) signal_child(PgArchPID, SIGQUIT); } = - /* - * Force a power-cycle of the pgstat process too. (This isn't absolutel= y - * necessary, but it seems like a good idea for robustness, and it - * simplifies the state-machine logic in the case where a shutdown reque= st - * arrives during crash processing.) - */ - if (PgStatPID !=3D 0 && take_action) - { - ereport(DEBUG2, - (errmsg_internal("sending %s to process %d", - "SIGQUIT", - (int) PgStatPID))); - signal_child(PgStatPID, SIGQUIT); - allow_immediate_pgstat_restart(); - } - /* We do NOT restart the syslogger */ = if (Shutdown !=3D ImmediateShutdown) @@ -3887,8 +3831,6 @@ PostmasterStateMachine(void) SignalChildren(SIGQUIT); if (PgArchPID !=3D 0) signal_child(PgArchPID, SIGQUIT); - if (PgStatPID !=3D 0) - signal_child(PgStatPID, SIGQUIT); } } } @@ -3923,8 +3865,7 @@ PostmasterStateMachine(void) * normal state transition leading up to PM_WAIT_DEAD_END, or during * FatalError processing. */ - if (dlist_is_empty(&BackendList) && - PgArchPID =3D=3D 0 && PgStatPID =3D=3D 0) + if (dlist_is_empty(&BackendList) && PgArchPID =3D=3D 0) { /* These other guys should be dead already */ Assert(StartupPID =3D=3D 0); @@ -4125,8 +4066,6 @@ TerminateChildren(int signal) signal_child(AutoVacPID, signal); if (PgArchPID !=3D 0) signal_child(PgArchPID, signal); - if (PgStatPID !=3D 0) - signal_child(PgStatPID, signal); } = /* @@ -5108,18 +5047,6 @@ SubPostmasterMain(int argc, char *argv[]) = StartBackgroundWorker(); } - if (strcmp(argv[1], "--forkarch") =3D=3D 0) - { - /* Do not want to attach to shared memory */ - - PgArchiverMain(argc, argv); /* does not return */ - } - if (strcmp(argv[1], "--forkcol") =3D=3D 0) - { - /* Do not want to attach to shared memory */ - - PgstatCollectorMain(argc, argv); /* does not return */ - } if (strcmp(argv[1], "--forklog") =3D=3D 0) { /* Do not want to attach to shared memory */ @@ -5238,12 +5165,6 @@ sigusr1_handler(SIGNAL_ARGS) if (CheckPostmasterSignal(PMSIGNAL_BEGIN_HOT_STANDBY) && pmState =3D=3D PM_RECOVERY && Shutdown =3D=3D NoShutdown) { - /* - * Likewise, start other special children as needed. - */ - Assert(PgStatPID =3D=3D 0); - PgStatPID =3D pgstat_start(); - ereport(LOG, (errmsg("database system is ready to accept read only connections")))= ; = @@ -6148,7 +6069,6 @@ extern slock_t *ShmemLock; extern slock_t *ProcStructLock; extern PGPROC *AuxiliaryProcs; extern PMSignalData *PMSignalState; -extern pgsocket pgStatSock; extern pg_time_t first_syslogger_file_time; = #ifndef WIN32 @@ -6204,8 +6124,6 @@ save_backend_variables(BackendParameters *param, Por= t *port, param->AuxiliaryProcs =3D AuxiliaryProcs; param->PreparedXactProcs =3D PreparedXactProcs; param->PMSignalState =3D PMSignalState; - if (!write_inheritable_socket(¶m->pgStatSock, pgStatSock, childPid)) - return false; = param->PostmasterPid =3D PostmasterPid; param->PgStartTime =3D PgStartTime; @@ -6440,7 +6358,6 @@ restore_backend_variables(BackendParameters *param, = Port *port) AuxiliaryProcs =3D param->AuxiliaryProcs; PreparedXactProcs =3D param->PreparedXactProcs; PMSignalState =3D param->PMSignalState; - read_inheritable_socket(&pgStatSock, ¶m->pgStatSock); = PostmasterPid =3D param->PostmasterPid; PgStartTime =3D param->PgStartTime; diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci= .c index 427b0d5..58a442f 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -147,6 +147,7 @@ CreateSharedMemoryAndSemaphores(void) size =3D add_size(size, BTreeShmemSize()); size =3D add_size(size, SyncScanShmemSize()); size =3D add_size(size, AsyncShmemSize()); + size =3D add_size(size, StatsShmemSize()); #ifdef EXEC_BACKEND size =3D add_size(size, ShmemBackendArraySize()); #endif @@ -263,6 +264,7 @@ CreateSharedMemoryAndSemaphores(void) BTreeShmemInit(); SyncScanShmemInit(); AsyncShmemInit(); + StatsShmemInit(); = #ifdef EXEC_BACKEND = diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/= lwlock.c index 4c14e51..162ef56 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -523,6 +523,7 @@ RegisterLWLockTranches(void) LWLockRegisterTranche(LWTRANCHE_PARALLEL_APPEND, "parallel_append"); LWLockRegisterTranche(LWTRANCHE_PARALLEL_HASH_JOIN, "parallel_hash_join"= ); LWLockRegisterTranche(LWTRANCHE_SXACT, "serializable_xact"); + LWLockRegisterTranche(LWTRANCHE_STATS, "activity stats"); = /* Register named tranches. */ for (i =3D 0; i < NamedLWLockTrancheRequests; i++) diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 9dba3b0..a10ec11 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3189,6 +3189,12 @@ ProcessInterrupts(void) = if (ParallelMessagePending) HandleParallelMessages(); + + if (IdleStatsUpdateTimeoutPending) + { + IdleStatsUpdateTimeoutPending =3D false; + pgstat_report_stat(true); + } } = = @@ -3763,6 +3769,7 @@ PostgresMain(int argc, char *argv[], sigjmp_buf local_sigjmp_buf; volatile bool send_ready_for_query =3D true; bool disable_idle_in_transaction_timeout =3D false; + bool disable_idle_stats_update_timeout =3D false; = /* Initialize startup process environment if necessary. */ if (!IsUnderPostmaster) @@ -4201,6 +4208,8 @@ PostgresMain(int argc, char *argv[], } else { + long stats_timeout; + /* Send out notify signals and transmit self-notifies */ ProcessCompletedNotifies(); = @@ -4213,8 +4222,13 @@ PostgresMain(int argc, char *argv[], if (notifyInterruptPending) ProcessNotifyInterrupt(); = - pgstat_report_stat(false); - + stats_timeout =3D pgstat_report_stat(false); + if (stats_timeout > 0) + { + disable_idle_stats_update_timeout =3D true; + enable_timeout_after(IDLE_STATS_UPDATE_TIMEOUT, + stats_timeout); + } set_ps_display("idle", false); pgstat_report_activity(STATE_IDLE, NULL); } @@ -4249,7 +4263,7 @@ PostgresMain(int argc, char *argv[], DoingCommandRead =3D false; = /* - * (5) turn off the idle-in-transaction timeout + * (5) turn off the idle-in-transaction timeout and stats update timeou= t */ if (disable_idle_in_transaction_timeout) { @@ -4257,6 +4271,12 @@ PostgresMain(int argc, char *argv[], disable_idle_in_transaction_timeout =3D false; } = + if (disable_idle_stats_update_timeout) + { + disable_timeout(IDLE_STATS_UPDATE_TIMEOUT, false); + disable_idle_stats_update_timeout =3D false; + } + /* * (6) check for any other interesting events that happened while we * slept. diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/glo= bals.c index eb19644..51748c9 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -33,6 +33,7 @@ volatile sig_atomic_t ProcDiePending =3D false; volatile sig_atomic_t ClientConnectionLost =3D false; volatile sig_atomic_t IdleInTransactionSessionTimeoutPending =3D false; volatile sig_atomic_t ProcSignalBarrierPending =3D false; +volatile sig_atomic_t IdleStatsUpdateTimeoutPending =3D false; volatile uint32 InterruptHoldoffCount =3D 0; volatile uint32 QueryCancelHoldoffCount =3D 0; volatile uint32 CritSectionCount =3D 0; diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/po= stinit.c index 8a47dcd..a9e5309 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -73,6 +73,7 @@ static void ShutdownPostgres(int code, Datum arg); static void StatementTimeoutHandler(void); static void LockTimeoutHandler(void); static void IdleInTransactionSessionTimeoutHandler(void); +static void IdleStatsUpdateTimeoutHandler(void); static bool ThereIsAtLeastOneRole(void); static void process_startup_options(Port *port, bool am_superuser); static void process_settings(Oid databaseid, Oid roleid); @@ -630,6 +631,8 @@ InitPostgres(const char *in_dbname, Oid dboid, const c= har *username, RegisterTimeout(LOCK_TIMEOUT, LockTimeoutHandler); RegisterTimeout(IDLE_IN_TRANSACTION_SESSION_TIMEOUT, IdleInTransactionSessionTimeoutHandler); + RegisterTimeout(IDLE_STATS_UPDATE_TIMEOUT, + IdleStatsUpdateTimeoutHandler); } = /* @@ -1240,6 +1243,14 @@ IdleInTransactionSessionTimeoutHandler(void) SetLatch(MyLatch); } = +static void +IdleStatsUpdateTimeoutHandler(void) +{ + IdleStatsUpdateTimeoutPending =3D true; + InterruptPending =3D true; + SetLatch(MyLatch); +} + /* * Returns true if at least one role is defined in this database cluster. */ diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_bas= ebackup/t/010_pg_basebackup.pl index 3c70499..927ae31 100644 --- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl +++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl @@ -6,7 +6,7 @@ use File::Basename qw(basename dirname); use File::Path qw(rmtree); use PostgresNode; use TestLib; -use Test::More tests =3D> 107; +use Test::More tests =3D> 106; = program_help_ok('pg_basebackup'); program_version_ok('pg_basebackup'); @@ -123,7 +123,7 @@ is_deeply( = # Contents of these directories should not be copied. foreach my $dirname ( - qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp = pg_subtrans) + qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_subtrans) ) { is_deeply( diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index ef4cb6a..96204b7 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -83,6 +83,8 @@ extern PGDLLIMPORT volatile sig_atomic_t QueryCancelPend= ing; extern PGDLLIMPORT volatile sig_atomic_t ProcDiePending; extern PGDLLIMPORT volatile sig_atomic_t IdleInTransactionSessionTimeoutP= ending; extern PGDLLIMPORT volatile sig_atomic_t ProcSignalBarrierPending; +extern PGDLLIMPORT volatile sig_atomic_t IdleStatsUpdateTimeoutPending; +extern PGDLLIMPORT volatile sig_atomic_t ConfigReloadPending; = extern PGDLLIMPORT volatile sig_atomic_t ClientConnectionLost; = diff --git a/src/include/pgstat.h b/src/include/pgstat.h index dfdac00..c415820 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -1,7 +1,7 @@ /* ---------- * pgstat.h * - * Definitions for the PostgreSQL statistics collector daemon. + * Definitions for the PostgreSQL statistics collector facility. * * Copyright (c) 2001-2020, PostgreSQL Global Development Group * @@ -13,10 +13,11 @@ = #include "datatype/timestamp.h" #include "libpq/pqcomm.h" -#include "port/atomics.h" +#include "lib/dshash.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" #include "storage/proc.h" +#include "storage/lwlock.h" #include "utils/hsearch.h" #include "utils/relcache.h" = @@ -41,33 +42,6 @@ typedef enum TrackFunctionsLevel } TrackFunctionsLevel; = /* ---------- - * The types of backend -> collector messages - * ---------- - */ -typedef enum StatMsgType -{ - PGSTAT_MTYPE_DUMMY, - PGSTAT_MTYPE_INQUIRY, - PGSTAT_MTYPE_TABSTAT, - PGSTAT_MTYPE_TABPURGE, - PGSTAT_MTYPE_DROPDB, - PGSTAT_MTYPE_RESETCOUNTER, - PGSTAT_MTYPE_RESETSHAREDCOUNTER, - PGSTAT_MTYPE_RESETSINGLECOUNTER, - PGSTAT_MTYPE_AUTOVAC_START, - PGSTAT_MTYPE_VACUUM, - PGSTAT_MTYPE_ANALYZE, - PGSTAT_MTYPE_ARCHIVER, - PGSTAT_MTYPE_BGWRITER, - PGSTAT_MTYPE_FUNCSTAT, - PGSTAT_MTYPE_FUNCPURGE, - PGSTAT_MTYPE_RECOVERYCONFLICT, - PGSTAT_MTYPE_TEMPFILE, - PGSTAT_MTYPE_DEADLOCK, - PGSTAT_MTYPE_CHECKSUMFAILURE -} StatMsgType; - -/* ---------- * The data type used for counters. * ---------- */ @@ -77,9 +51,8 @@ typedef int64 PgStat_Counter; * PgStat_TableCounts The actual per-table counts kept by a backend * * This struct should contain only actual event counters, because we memc= mp - * it against zeroes to detect whether there are any counts to transmit. - * It is a component of PgStat_TableStatus (within-backend state) and - * PgStat_TableEntry (the transmitted message format). + * it against zeroes to detect whether there are any counts to write. + * It is a component of PgStat_TableStatus (within-backend state). * * Note: for a table, tuples_returned is the number of tuples successfull= y * fetched by heap_getnext, while tuples_fetched is the number of tuples @@ -115,13 +88,6 @@ typedef struct PgStat_TableCounts PgStat_Counter t_blocks_hit; } PgStat_TableCounts; = -/* Possible targets for resetting cluster-wide shared values */ -typedef enum PgStat_Shared_Reset_Target -{ - RESET_ARCHIVER, - RESET_BGWRITER -} PgStat_Shared_Reset_Target; - /* Possible object types for resetting single counters */ typedef enum PgStat_Single_Reset_Type { @@ -180,236 +146,12 @@ typedef struct PgStat_TableXactStatus } PgStat_TableXactStatus; = = -/* ------------------------------------------------------------ - * Message formats follow - * ------------------------------------------------------------ - */ - - -/* ---------- - * PgStat_MsgHdr The common message header - * ---------- - */ -typedef struct PgStat_MsgHdr -{ - StatMsgType m_type; - int m_size; -} PgStat_MsgHdr; - -/* ---------- - * Space available in a message. This will keep the UDP packets below 1K= , - * which should fit unfragmented into the MTU of the loopback interface. - * (Larger values of PGSTAT_MAX_MSG_SIZE would work for that on most - * platforms, but we're being conservative here.) - * ---------- - */ -#define PGSTAT_MAX_MSG_SIZE 1000 -#define PGSTAT_MSG_PAYLOAD (PGSTAT_MAX_MSG_SIZE - sizeof(PgStat_MsgHdr)) - - -/* ---------- - * PgStat_MsgDummy A dummy message, ignored by the collector - * ---------- - */ -typedef struct PgStat_MsgDummy -{ - PgStat_MsgHdr m_hdr; -} PgStat_MsgDummy; - - -/* ---------- - * PgStat_MsgInquiry Sent by a backend to ask the collector - * to write the stats file(s). - * - * Ordinarily, an inquiry message prompts writing of the global stats fil= e, - * the stats file for shared catalogs, and the stats file for the specifi= ed - * database. If databaseid is InvalidOid, only the first two are written= . - * - * New file(s) will be written only if the existing file has a timestamp - * older than the specified cutoff_time; this prevents duplicated effort - * when multiple requests arrive at nearly the same time, assuming that - * backends send requests with cutoff_times a little bit in the past. - * - * clock_time should be the requestor's current local time; the collector - * uses this to check for the system clock going backward, but it has no - * effect unless that occurs. We assume clock_time >=3D cutoff_time, tho= ugh. - * ---------- - */ - -typedef struct PgStat_MsgInquiry -{ - PgStat_MsgHdr m_hdr; - TimestampTz clock_time; /* observed local clock time */ - TimestampTz cutoff_time; /* minimum acceptable file timestamp */ - Oid databaseid; /* requested DB (InvalidOid =3D> shared only) */ -} PgStat_MsgInquiry; - - -/* ---------- - * PgStat_TableEntry Per-table info in a MsgTabstat - * ---------- - */ -typedef struct PgStat_TableEntry -{ - Oid t_id; - PgStat_TableCounts t_counts; -} PgStat_TableEntry; - -/* ---------- - * PgStat_MsgTabstat Sent by the backend to report table - * and buffer access statistics. - * ---------- - */ -#define PGSTAT_NUM_TABENTRIES \ - ((PGSTAT_MSG_PAYLOAD - sizeof(Oid) - 3 * sizeof(int) - 2 * sizeof(PgStat= _Counter)) \ - / sizeof(PgStat_TableEntry)) - -typedef struct PgStat_MsgTabstat -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - int m_nentries; - int m_xact_commit; - int m_xact_rollback; - PgStat_Counter m_block_read_time; /* times in microseconds */ - PgStat_Counter m_block_write_time; - PgStat_TableEntry m_entry[PGSTAT_NUM_TABENTRIES]; -} PgStat_MsgTabstat; - - -/* ---------- - * PgStat_MsgTabpurge Sent by the backend to tell the collector - * about dead tables. - * ---------- - */ -#define PGSTAT_NUM_TABPURGE \ - ((PGSTAT_MSG_PAYLOAD - sizeof(Oid) - sizeof(int)) \ - / sizeof(Oid)) - -typedef struct PgStat_MsgTabpurge -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - int m_nentries; - Oid m_tableid[PGSTAT_NUM_TABPURGE]; -} PgStat_MsgTabpurge; - - -/* ---------- - * PgStat_MsgDropdb Sent by the backend to tell the collector - * about a dropped database - * ---------- - */ -typedef struct PgStat_MsgDropdb -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; -} PgStat_MsgDropdb; - - -/* ---------- - * PgStat_MsgResetcounter Sent by the backend to tell the collector - * to reset counters - * ---------- - */ -typedef struct PgStat_MsgResetcounter -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; -} PgStat_MsgResetcounter; - -/* ---------- - * PgStat_MsgResetsharedcounter Sent by the backend to tell the collector - * to reset a shared counter - * ---------- - */ -typedef struct PgStat_MsgResetsharedcounter -{ - PgStat_MsgHdr m_hdr; - PgStat_Shared_Reset_Target m_resettarget; -} PgStat_MsgResetsharedcounter; - -/* ---------- - * PgStat_MsgResetsinglecounter Sent by the backend to tell the collector - * to reset a single counter - * ---------- - */ -typedef struct PgStat_MsgResetsinglecounter -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - PgStat_Single_Reset_Type m_resettype; - Oid m_objectid; -} PgStat_MsgResetsinglecounter; - -/* ---------- - * PgStat_MsgAutovacStart Sent by the autovacuum daemon to signal - * that a database is going to be processed - * ---------- - */ -typedef struct PgStat_MsgAutovacStart -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - TimestampTz m_start_time; -} PgStat_MsgAutovacStart; - - /* ---------- - * PgStat_MsgVacuum Sent by the backend or autovacuum daemon - * after VACUUM - * ---------- - */ -typedef struct PgStat_MsgVacuum -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - Oid m_tableoid; - bool m_autovacuum; - TimestampTz m_vacuumtime; - PgStat_Counter m_live_tuples; - PgStat_Counter m_dead_tuples; -} PgStat_MsgVacuum; - - -/* ---------- - * PgStat_MsgAnalyze Sent by the backend or autovacuum daemon - * after ANALYZE - * ---------- - */ -typedef struct PgStat_MsgAnalyze -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - Oid m_tableoid; - bool m_autovacuum; - bool m_resetcounter; - TimestampTz m_analyzetime; - PgStat_Counter m_live_tuples; - PgStat_Counter m_dead_tuples; -} PgStat_MsgAnalyze; - - -/* ---------- - * PgStat_MsgArchiver Sent by the archiver to update statistics. - * ---------- - */ -typedef struct PgStat_MsgArchiver -{ - PgStat_MsgHdr m_hdr; - bool m_failed; /* Failed attempt */ - char m_xlog[MAX_XFN_CHARS + 1]; - TimestampTz m_timestamp; -} PgStat_MsgArchiver; - -/* ---------- - * PgStat_MsgBgWriter Sent by the bgwriter to update statistics. + * PgStat_MsgBgWriter bgwriter statistics * ---------- */ typedef struct PgStat_MsgBgWriter { - PgStat_MsgHdr m_hdr; - PgStat_Counter m_timed_checkpoints; PgStat_Counter m_requested_checkpoints; PgStat_Counter m_buf_written_checkpoints; @@ -423,37 +165,13 @@ typedef struct PgStat_MsgBgWriter } PgStat_MsgBgWriter; = /* ---------- - * PgStat_MsgRecoveryConflict Sent by the backend upon recovery conflict - * ---------- - */ -typedef struct PgStat_MsgRecoveryConflict -{ - PgStat_MsgHdr m_hdr; - - Oid m_databaseid; - int m_reason; -} PgStat_MsgRecoveryConflict; - -/* ---------- - * PgStat_MsgTempFile Sent by the backend upon creating a temp file - * ---------- - */ -typedef struct PgStat_MsgTempFile -{ - PgStat_MsgHdr m_hdr; - - Oid m_databaseid; - size_t m_filesize; -} PgStat_MsgTempFile; - -/* ---------- * PgStat_FunctionCounts The actual per-function counts kept by a backend * * This struct should contain only actual event counters, because we memc= mp - * it against zeroes to detect whether there are any counts to transmit. + * it against zeroes to detect whether there are any counts to write. * * Note that the time counters are in instr_time format here. We convert= to - * microseconds in PgStat_Counter format when transmitting to the collect= or. + * microseconds in PgStat_Counter format when writing to shared statsitic= s. * ---------- */ typedef struct PgStat_FunctionCounts @@ -485,96 +203,8 @@ typedef struct PgStat_FunctionEntry PgStat_Counter f_self_time; } PgStat_FunctionEntry; = -/* ---------- - * PgStat_MsgFuncstat Sent by the backend to report function - * usage statistics. - * ---------- - */ -#define PGSTAT_NUM_FUNCENTRIES \ - ((PGSTAT_MSG_PAYLOAD - sizeof(Oid) - sizeof(int)) \ - / sizeof(PgStat_FunctionEntry)) - -typedef struct PgStat_MsgFuncstat -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - int m_nentries; - PgStat_FunctionEntry m_entry[PGSTAT_NUM_FUNCENTRIES]; -} PgStat_MsgFuncstat; - -/* ---------- - * PgStat_MsgFuncpurge Sent by the backend to tell the collector - * about dead functions. - * ---------- - */ -#define PGSTAT_NUM_FUNCPURGE \ - ((PGSTAT_MSG_PAYLOAD - sizeof(Oid) - sizeof(int)) \ - / sizeof(Oid)) - -typedef struct PgStat_MsgFuncpurge -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - int m_nentries; - Oid m_functionid[PGSTAT_NUM_FUNCPURGE]; -} PgStat_MsgFuncpurge; - -/* ---------- - * PgStat_MsgDeadlock Sent by the backend to tell the collector - * about a deadlock that occurred. - * ---------- - */ -typedef struct PgStat_MsgDeadlock -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; -} PgStat_MsgDeadlock; - -/* ---------- - * PgStat_MsgChecksumFailure Sent by the backend to tell the collector - * about checksum failures noticed. - * ---------- - */ -typedef struct PgStat_MsgChecksumFailure -{ - PgStat_MsgHdr m_hdr; - Oid m_databaseid; - int m_failurecount; - TimestampTz m_failure_time; -} PgStat_MsgChecksumFailure; - - -/* ---------- - * PgStat_Msg Union over all possible messages. - * ---------- - */ -typedef union PgStat_Msg -{ - PgStat_MsgHdr msg_hdr; - PgStat_MsgDummy msg_dummy; - PgStat_MsgInquiry msg_inquiry; - PgStat_MsgTabstat msg_tabstat; - PgStat_MsgTabpurge msg_tabpurge; - PgStat_MsgDropdb msg_dropdb; - PgStat_MsgResetcounter msg_resetcounter; - PgStat_MsgResetsharedcounter msg_resetsharedcounter; - PgStat_MsgResetsinglecounter msg_resetsinglecounter; - PgStat_MsgAutovacStart msg_autovacuum_start; - PgStat_MsgVacuum msg_vacuum; - PgStat_MsgAnalyze msg_analyze; - PgStat_MsgArchiver msg_archiver; - PgStat_MsgBgWriter msg_bgwriter; - PgStat_MsgFuncstat msg_funcstat; - PgStat_MsgFuncpurge msg_funcpurge; - PgStat_MsgRecoveryConflict msg_recoveryconflict; - PgStat_MsgDeadlock msg_deadlock; - PgStat_MsgTempFile msg_tempfile; - PgStat_MsgChecksumFailure msg_checksumfailure; -} PgStat_Msg; - - /* ------------------------------------------------------------ - * Statistic collector data structures follow + * Statistic collector data structures on file and shared memory follow * * PGSTAT_FILE_FORMAT_ID should be changed whenever any of these * data structures change. @@ -614,16 +244,29 @@ typedef struct PgStat_StatDBEntry PgStat_Counter n_block_write_time; = TimestampTz stat_reset_timestamp; - TimestampTz stats_timestamp; /* time of db stats file update */ + TimestampTz stats_timestamp; /* time of db stats update */ = /* - * tables and functions must be last in the struct, because we don't wri= te - * the pointers out to the stats file. + * The followings must be last in the struct, because we don't write the= m + * out to the stats file. */ - HTAB *tables; - HTAB *functions; + int generation; /* current generation of the below */ + int refcnt; /* current gen reference count */ + dshash_table_handle tables; /* current gen tables hash */ + dshash_table_handle functions; /* current gen functions hash */ + int prev_refcnt; /* prev gen reference count */ + dshash_table_handle prev_tables; /* prev gen tables hash */ + dshash_table_handle prev_functions; /* prev gen functions hash */ + LWLock lock; /* Lock for the above members */ + + /* non-shared members */ + HTAB *snapshot_tables; /* table entry snapshot */ + HTAB *snapshot_functions; /* function entry snapshot */ + dshash_table *dshash_tables; /* attached tables dshash */ + dshash_table *dshash_functions; /* attached functions dshash */ } PgStat_StatDBEntry; = +#define SHARED_DBENT_SIZE offsetof(PgStat_StatDBEntry, snapshot_tables) = /* ---------- * PgStat_StatTabEntry The collector's data per table (or index) @@ -662,7 +305,7 @@ typedef struct PgStat_StatTabEntry = = /* ---------- - * PgStat_StatFuncEntry The collector's data per function + * PgStat_StatFuncEntry per function stats data * ---------- */ typedef struct PgStat_StatFuncEntry @@ -677,7 +320,7 @@ typedef struct PgStat_StatFuncEntry = = /* - * Archiver statistics kept in the stats collector + * Archiver statistics kept in the shared stats */ typedef struct PgStat_ArchiverStats { @@ -693,7 +336,7 @@ typedef struct PgStat_ArchiverStats } PgStat_ArchiverStats; = /* - * Global statistics kept in the stats collector + * Global statistics kept in the shared stats */ typedef struct PgStat_GlobalStats { @@ -779,7 +422,6 @@ typedef enum WAIT_EVENT_CHECKPOINTER_MAIN, WAIT_EVENT_LOGICAL_APPLY_MAIN, WAIT_EVENT_LOGICAL_LAUNCHER_MAIN, - WAIT_EVENT_PGSTAT_MAIN, WAIT_EVENT_RECOVERY_WAL_ALL, WAIT_EVENT_RECOVERY_WAL_STREAM, WAIT_EVENT_SYSLOGGER_MAIN, @@ -1217,6 +859,8 @@ extern PGDLLIMPORT bool pgstat_track_counts; extern PGDLLIMPORT int pgstat_track_functions; extern PGDLLIMPORT int pgstat_track_activity_query_size; extern char *pgstat_stat_directory; + +/* No longer used, but will be removed with GUC */ extern char *pgstat_stat_tmpname; extern char *pgstat_stat_filename; = @@ -1238,29 +882,26 @@ extern PgStat_Counter pgStatBlockWriteTime; extern Size BackendStatusShmemSize(void); extern void CreateSharedBackendStatus(void); = -extern void pgstat_init(void); -extern int pgstat_start(void); -extern void pgstat_reset_all(void); -extern void allow_immediate_pgstat_restart(void); +extern Size StatsShmemSize(void); +extern void StatsShmemInit(void); = -#ifdef EXEC_BACKEND -extern void PgstatCollectorMain(int argc, char *argv[]) pg_attribute_nore= turn(); -#endif +extern void pgstat_reset_all(void); = +/* File input/output functions */ +extern void pgstat_read_statsfiles(void); +extern void pgstat_write_statsfiles(void); = /* ---------- * Functions called from backends * ---------- */ -extern void pgstat_ping(void); - -extern void pgstat_report_stat(bool force); +extern long pgstat_report_stat(bool force); extern void pgstat_vacuum_stat(void); extern void pgstat_drop_database(Oid databaseid); = extern void pgstat_clear_snapshot(void); extern void pgstat_reset_counters(void); -extern void pgstat_reset_shared_counters(const char *); +extern void pgstat_reset_shared_counters(const char *target); extern void pgstat_reset_single_counter(Oid objectid, PgStat_Single_Reset= _Type type); = extern void pgstat_report_autovac(Oid dboid); @@ -1432,11 +1073,13 @@ extern void pgstat_send_bgwriter(void); */ extern PgStat_StatDBEntry *pgstat_fetch_stat_dbentry(Oid dbid); extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry(Oid relid); +extern PgStat_StatTabEntry *pgstat_fetch_stat_tabentry_extended(PgStat_St= atDBEntry *dbent, Oid relid); extern PgBackendStatus *pgstat_fetch_stat_beentry(int beid); extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid); extern PgStat_StatFuncEntry *pgstat_fetch_stat_funcentry(Oid funcid); extern int pgstat_fetch_stat_numbackends(void); extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); extern PgStat_GlobalStats *pgstat_fetch_global(void); +extern void pgstat_clear_snapshot(void); = #endif /* PGSTAT_H */ diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 8fda8e4..13371e8 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -220,6 +220,7 @@ typedef enum BuiltinTrancheIds LWTRANCHE_TBM, LWTRANCHE_PARALLEL_APPEND, LWTRANCHE_SXACT, + LWTRANCHE_STATS, LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; = diff --git a/src/include/utils/timeout.h b/src/include/utils/timeout.h index 83a15f6..77d1572 100644 --- a/src/include/utils/timeout.h +++ b/src/include/utils/timeout.h @@ -31,6 +31,7 @@ typedef enum TimeoutId STANDBY_TIMEOUT, STANDBY_LOCK_TIMEOUT, IDLE_IN_TRANSACTION_SESSION_TIMEOUT, + IDLE_STATS_UPDATE_TIMEOUT, /* First user-definable timeout reason */ USER_TIMEOUT, /* Maximum number of timeout reasons */ ------- =_aaaaaaaaaa0 Content-Type: text/x-diff; name="0005-Remove-the-GUC-stats_temp_directory.patch"; charset="us-ascii" Content-ID: <15877.1583770863.6@sss.pgh.pa.us> Content-Description: 0005-Remove-the-GUC-stats_temp_directory.patch Content-Transfer-Encoding: quoted-printable diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml index bdc9026..2885540 100644 --- a/doc/src/sgml/backup.sgml +++ b/doc/src/sgml/backup.sgml @@ -1146,8 +1146,6 @@ SELECT pg_stop_backup(); pg_snapshots/, pg_stat_tmp/= , and pg_subtrans/ (but not the directories themse= lves) can be omitted from the backup as they will be initialized on postmaster sta= rtup. - If is set and is under t= he data - directory then the contents of that directory can also be omitted. = diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index c1128f8..8b63b79 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7052,25 +7052,6 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' = WITH csv; = - - stats_temp_directory (string) - - stats_temp_directory configuration par= ameter - - - - - Sets the directory to store temporary statistics data in. This ca= n be - a path relative to the data directory or an absolute path. The de= fault - is pg_stat_tmp. Pointing this at a RAM-based - file system will decrease physical I/O requirements and can lead = to - improved performance. - This parameter can only be set in the postgresql.conf - file or on the server command line. - - - - = diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 11aaef5..0f8aa93 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -195,12 +195,7 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss = 11:34 0:00 postgres: ser = The statistics collector transmits the collected information to other - PostgreSQL processes through temporary file= s. - These files are stored in the directory named by the - parameter, - pg_stat_tmp by default. - For better performance, stats_temp_directory can be - pointed at a RAM-based file system, decreasing physical I/O requiremen= ts. + PostgreSQL processes through shared memory. When the server shuts down cleanly, a permanent copy of the statistics data is stored in the pg_stat subdirectory, so th= at statistics can be retained across server restarts. When recovery is diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 1c19e86..2f04bb6 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -122,8 +122,7 @@ Item = pg_stat_tmp - Subdirectory containing temporary files for the statistics - subsystem + Subdirectory containing ephemeral files for extensions = diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgst= at.c index 8ea8e52..4ba2460 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -110,15 +110,12 @@ bool pgstat_track_counts =3D false; int pgstat_track_functions =3D TRACK_FUNC_OFF; int pgstat_track_activity_query_size =3D 1024; = -/* ---------- - * Built from GUC parameter - * ---------- +/* + * This used to be a GUC variable and is no longer used in this file, but= left + * alone just for backward comptibility for extensions, having the defaul= t + * value. */ -char *pgstat_stat_directory =3D NULL; - -/* No longer used, but will be removed with GUC */ -char *pgstat_stat_filename =3D NULL; -char *pgstat_stat_tmpname =3D NULL; +char *pgstat_stat_directory =3D PG_STAT_TMP_DIR; = #define StatsLock (&StatsShmem->StatsMainLock) = diff --git a/src/backend/replication/basebackup.c b/src/backend/replicatio= n/basebackup.c index f66cbc2..fa66bef 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -251,15 +251,12 @@ perform_base_backup(basebackup_options *opt) TimeLineID endtli; StringInfo labelfile; StringInfo tblspc_map_file =3D NULL; - int datadirpathlen; List *tablespaces =3D NIL; = backup_total =3D 0; backup_streamed =3D 0; pgstat_progress_start_command(PROGRESS_COMMAND_BASEBACKUP, InvalidOid); = - datadirpathlen =3D strlen(DataDir); - backup_started_in_recovery =3D RecoveryInProgress(); = labelfile =3D makeStringInfo(); @@ -291,13 +288,9 @@ perform_base_backup(basebackup_options *opt) * Calculate the relative path of temporary statistics directory in * order to skip the files which are located in that directory later. */ - if (is_absolute_path(pgstat_stat_directory) && - strncmp(pgstat_stat_directory, DataDir, datadirpathlen) =3D=3D 0) - statrelpath =3D psprintf("./%s", pgstat_stat_directory + datadirpathle= n + 1); - else if (strncmp(pgstat_stat_directory, "./", 2) !=3D 0) - statrelpath =3D psprintf("./%s", pgstat_stat_directory); - else - statrelpath =3D pgstat_stat_directory; + + Assert(strchr(PG_STAT_TMP_DIR, '/') =3D=3D NULL); + statrelpath =3D psprintf("./%s", PG_STAT_TMP_DIR); = /* Add a node for the base directory at the end */ ti =3D palloc0(sizeof(tablespaceinfo)); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index c1fad3b..66efb33 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -196,7 +196,6 @@ static bool check_max_wal_senders(int *newval, void **= extra, GucSource source); static bool check_autovacuum_work_mem(int *newval, void **extra, GucSourc= e source); static bool check_effective_io_concurrency(int *newval, void **extra, Guc= Source source); static void assign_effective_io_concurrency(int newval, void *extra); -static void assign_pgstat_temp_directory(const char *newval, void *extra)= ; static bool check_application_name(char **newval, void **extra, GucSource= source); static void assign_application_name(const char *newval, void *extra); static bool check_cluster_name(char **newval, void **extra, GucSource sou= rce); @@ -4193,17 +4192,6 @@ static struct config_string ConfigureNamesString[] = =3D }, = { - {"stats_temp_directory", PGC_SIGHUP, STATS_COLLECTOR, - gettext_noop("Writes temporary statistics files to the specified direc= tory."), - NULL, - GUC_SUPERUSER_ONLY - }, - &pgstat_temp_directory, - PG_STAT_TMP_DIR, - check_canonical_path, assign_pgstat_temp_directory, NULL - }, - - { {"synchronous_standby_names", PGC_SIGHUP, REPLICATION_MASTER, gettext_noop("Number of synchronous standbys and list of names of pote= ntial synchronous ones."), NULL, @@ -11488,35 +11476,6 @@ assign_effective_io_concurrency(int newval, void = *extra) #endif /* USE_PREFETCH */ } = -static void -assign_pgstat_temp_directory(const char *newval, void *extra) -{ - /* check_canonical_path already canonicalized newval for us */ - char *dname; - char *tname; - char *fname; - - /* directory */ - dname =3D guc_malloc(ERROR, strlen(newval) + 1); /* runtime dir */ - sprintf(dname, "%s", newval); - - /* global stats */ - tname =3D guc_malloc(ERROR, strlen(newval) + 12); /* /global.tmp */ - sprintf(tname, "%s/global.tmp", newval); - fname =3D guc_malloc(ERROR, strlen(newval) + 13); /* /global.stat */ - sprintf(fname, "%s/global.stat", newval); - - if (pgstat_stat_directory) - free(pgstat_stat_directory); - pgstat_stat_directory =3D dname; - if (pgstat_stat_tmpname) - free(pgstat_stat_tmpname); - pgstat_stat_tmpname =3D tname; - if (pgstat_stat_filename) - free(pgstat_stat_filename); - pgstat_stat_filename =3D fname; -} - static bool check_application_name(char **newval, void **extra, GucSource source) { diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/u= tils/misc/postgresql.conf.sample index e58e478..99f8513 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -572,7 +572,6 @@ #track_io_timing =3D off #track_functions =3D none # none, pl, all #track_activity_query_size =3D 1024 # (change requires restart) -#stats_temp_directory =3D 'pg_stat_tmp' = = # - Monitoring - diff --git a/src/include/pgstat.h b/src/include/pgstat.h index c415820..a1f1ad8 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -30,7 +30,10 @@ #define PGSTAT_STAT_PERMANENT_FILENAME "pg_stat/global.stat" #define PGSTAT_STAT_PERMANENT_TMPFILE "pg_stat/global.tmp" = -/* Default directory to store temporary statistics data in */ +/* + * This used to be the directory to store temporary statistics data in bu= t is + * no longer used. Defined here for backward compatibility. + */ #define PG_STAT_TMP_DIR "pg_stat_tmp" = /* Values for track_functions GUC variable --- order is significant! */ diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm index 9575268..f3340f7 100644 --- a/src/test/perl/PostgresNode.pm +++ b/src/test/perl/PostgresNode.pm @@ -455,10 +455,6 @@ sub init print $conf TestLib::slurp_file($ENV{TEMP_CONFIG}) if defined $ENV{TEMP_CONFIG}; = - # XXX Neutralize any stats_temp_directory in TEMP_CONFIG. Nodes running - # concurrently must not share a stats_temp_directory. - print $conf "stats_temp_directory =3D 'pg_stat_tmp'\n"; - if ($params{allows_streaming}) { if ($params{allows_streaming} eq "logical") ------- =_aaaaaaaaaa0--