agora inbox for [email protected]
help / color / mirror / Atom feed[PATCH 06/17] Allow user to use password instead of encryption key.
33+ messages / 8 participants
[nested] [flat]
* [PATCH 06/17] Allow user to use password instead of encryption key.
@ 2019-07-05 14:24 Antonin Houska <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Antonin Houska @ 2019-07-05 14:24 UTC (permalink / raw)
pg_keytool reads the password from stdin, uses "key derivation function (KDF)
parameters" stored in the data directory to derive the key and writes the key
to standard output.
The cluster can be created this way
initdb -D data -K "echo securepwd | pg_keytool -D data -w"
and started either this way
pg_ctl -D data -K "echo securepwd | pg_keytool -D data -w" start
or this way
postgres -D data
and (in another session)
echo securepwd | pg_keytool -D data -ws
Both initdb and pg_ctl can substitute the data directory for %D pattern in the
key encryption command. Thus user uses pg_keytool to derive the password, he
can in fact say
initdb -D data -K "echo securepwd | pg_keytool -D %D -w"
and
pg_ctl -D data -K "echo securepwd | pg_keytool -D %D -w" start
respectively.
---
src/bin/initdb/initdb.c | 26 +++-
src/bin/pg_ctl/pg_ctl.c | 2 +-
src/bin/pg_keytool/pg_keytool.c | 87 +++++++++---
src/fe_utils/encryption.c | 273 +++++++++++++++++++++++++++++++++++++-
src/include/fe_utils/encryption.h | 8 +-
src/include/storage/encryption.h | 8 ++
6 files changed, 379 insertions(+), 25 deletions(-)
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index f2a582e975..bfa8f5fad4 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -3004,11 +3004,31 @@ initialize_data_directory(void)
write_version_file(NULL);
/*
- * If the cluster will be encrypted, run the command to generate the
- * encryption key.
+ * If the cluster will be encrypted, write the KDF file so that encryption
+ * key can be derived from password.
*/
if (encryption_key_command)
- run_encryption_key_command(encryption_key);
+ {
+ /*
+ * XXX Since execution of encryption_key_command produce the key (as
+ * opposed to password), we don't know if the command received the key
+ * itself or a password. If DBA provided initdb with a key, he will
+ * never use password in the future (there was no KDF so far so the
+ * key could not be derived from password, and the password can hardly
+ * be derived from the key), so the KDF file may be useless. We don't
+ * have enough information to recognize this special case, so just
+ * initialize and write the KDF unconditionally.
+ */
+ init_kdf();
+ write_kdf_file(pg_data);
+
+ /*
+ * The key command is allowed to use pg_keytool, which in turn needs
+ * the KDF parameters. The KDF parameters are now available so we can
+ * run the command.
+ */
+ run_encryption_key_command(encryption_key, pg_data);
+ }
/* Select suitable configuration settings */
set_null_conf();
diff --git a/src/bin/pg_ctl/pg_ctl.c b/src/bin/pg_ctl/pg_ctl.c
index 7e414ac048..e8e4eee162 100644
--- a/src/bin/pg_ctl/pg_ctl.c
+++ b/src/bin/pg_ctl/pg_ctl.c
@@ -881,7 +881,7 @@ do_start(void)
* If encryption key is needed, retrieve it before trying to start
* postmaster.
*/
- run_encryption_key_command(encryption_key);
+ run_encryption_key_command(encryption_key, pg_data);
/*
* Where should the key be sent?
diff --git a/src/bin/pg_keytool/pg_keytool.c b/src/bin/pg_keytool/pg_keytool.c
index 322625da41..c5151ad57c 100644
--- a/src/bin/pg_keytool/pg_keytool.c
+++ b/src/bin/pg_keytool/pg_keytool.c
@@ -43,6 +43,7 @@ usage(const char *progname)
printf(_("Usage:\n"));
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions:\n"));
+ printf(_(" -D, --pgdata=DATADIR data directory\n"));
/* Display default host */
env = getenv("PGHOST");
printf(_(" -h, --host=HOSTNAME database server host or socket directory (default: \"%s\")\n"),
@@ -52,8 +53,9 @@ usage(const char *progname)
printf(_(" -p, --port=PORT database server port (default: \"%s\")\n"),
env ? env : DEF_PGPORT_STR);
printf(_(" -s, send output to database server\n"));
+ printf(_(" -w expect password on input, not a key\n"));
printf(_(" -?, --help show this help, then exit\n\n"));
- printf(_("Key is read from stdin and sent either to stdout or to PostgreSQL server being started\n"));
+ printf(_("Password or key is read from stdin. Key is sent to PostgreSQL server being started\n"));
}
#endif /* USE_ENCRYPTION */
@@ -69,9 +71,12 @@ main(int argc, char **argv)
int c;
char *host = NULL;
char *port_str = NULL;
+ char *DataDir = NULL;
bool to_server = false;
+ bool expect_password = false;
int i, n;
int optindex;
+ char password[ENCRYPTION_PWD_MAX_LENGTH];
char key_chars[ENCRYPTION_KEY_CHARS];
static struct option long_options[] =
@@ -99,11 +104,15 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "h:p:s",
+ while ((c = getopt_long(argc, argv, "h:D:p:sw",
long_options, &optindex)) != -1)
{
switch (c)
{
+ case 'D':
+ DataDir = optarg;
+ break;
+
case 'h':
host = pg_strdup(optarg);
break;
@@ -116,6 +125,10 @@ main(int argc, char **argv)
to_server = true;
break;
+ case 'w':
+ expect_password = true;
+ break;
+
case '?':
/* Actual help option given */
if (strcmp(argv[optind - 1], "-?") == 0)
@@ -146,34 +159,78 @@ main(int argc, char **argv)
exit(1);
}
- /* Read the key. */
+ /*
+ * The KDF file is needed to derive the key from password, and this file
+ * is located in the data directory.
+ */
+ if (expect_password && DataDir == NULL)
+ {
+ pg_log_error("%s: no data directory specified", progname);
+ pg_log_error("Try \"%s --help\" for more information.", progname);
+ exit(EXIT_FAILURE);
+ }
+
+ /*
+ * Read the credentials (key or password).
+ */
n = 0;
/* Key length in characters (two characters per hexadecimal digit) */
while ((c = getchar()) != EOF && c != '\n')
{
- if (n >= ENCRYPTION_KEY_CHARS)
+ if (!expect_password)
{
- pg_log_error("The key is too long");
- exit(EXIT_FAILURE);
+ if (n >= ENCRYPTION_KEY_CHARS)
+ {
+ pg_log_error("The key is too long");
+ exit(EXIT_FAILURE);
+ }
+
+ key_chars[n++] = c;
}
+ else
+ {
+ if (n >= ENCRYPTION_PWD_MAX_LENGTH)
+ {
+ pg_log_error("The password is too long");
+ exit(EXIT_FAILURE);
+ }
- key_chars[n++] = c;
+ password[n++] = c;
+ }
}
- if (n < ENCRYPTION_KEY_CHARS)
+ /* If password was received, turn it into encryption key. */
+ if (!expect_password)
{
- pg_log_error("The key is too short");
- exit(EXIT_FAILURE);
- }
+ if (n < ENCRYPTION_KEY_CHARS)
+ {
+ pg_log_error("The key is too short");
+ exit(EXIT_FAILURE);
+ }
- for (i = 0; i < ENCRYPTION_KEY_LENGTH; i++)
+ for (i = 0; i < ENCRYPTION_KEY_LENGTH; i++)
+ {
+ if (sscanf(key_chars + 2 * i, "%2hhx", encryption_key + i) == 0)
+ {
+ pg_log_error("Invalid character in encryption key at position %d",
+ 2 * i);
+ exit(EXIT_FAILURE);
+ }
+ }
+ }
+ else
{
- if (sscanf(key_chars + 2 * i, "%2hhx", encryption_key + i) == 0)
+ if (n < ENCRYPTION_PWD_MIN_LENGTH)
{
- pg_log_error("Invalid character in encryption key at position %d",
- 2 * i);
+ pg_log_error("The password is too short");
exit(EXIT_FAILURE);
}
+
+ /* Read the KDF parameters. */
+ read_kdf_file(DataDir);
+
+ /* Run the KDF. */
+ derive_key_from_password(encryption_key, password, n);
}
/*
diff --git a/src/fe_utils/encryption.c b/src/fe_utils/encryption.c
index ca37c9f373..134b3bde9b 100644
--- a/src/fe_utils/encryption.c
+++ b/src/fe_utils/encryption.c
@@ -21,31 +21,294 @@
#include "common/file_perm.h"
#include "common/logging.h"
#include "fe_utils/encryption.h"
-#include "storage/encryption.h"
#include "libpq-fe.h"
#include "libpq-int.h"
#include "libpq/pqcomm.h"
char *encryption_key_command = NULL;
+#define KDF_PARAMS_FILE "global/kdf_params"
+#define KDF_PARAMS_FILE_SIZE 512
+
+/*
+ * Key derivation function.
+ */
+typedef enum KDFKind
+{
+ KDF_OPENSSL_PKCS5_PBKDF2_HMAC_SHA = 0
+} KFDKind;
+
+typedef struct KDFParamsPBKDF2
+{
+ unsigned long int niter;
+ unsigned char salt[ENCRYPTION_KDF_SALT_LEN];
+} KDFParamsPBKDF2;
+
+/*
+ * Parameters of the key derivation function.
+ *
+ * The parameters are generated by initdb and stored into a file, which is
+ * then read during PG startup. This is similar to storing various settings in
+ * pg_control. However an existing KDF file is read only, so it does not have
+ * to be stored in shared memory.
+ */
+typedef struct KDFParamsData
+{
+ KFDKind function;
+
+ /*
+ * Function-specific parameters.
+ */
+ union
+ {
+ KDFParamsPBKDF2 pbkdf2;
+ } data;
+
+ /* CRC of all above ... MUST BE LAST! */
+ pg_crc32c crc;
+} KDFParamsData;
+
+extern KDFParamsData *KDFParams;
+
+/*
+ * Pointer to the KDF parameters.
+ */
+KDFParamsData *KDFParams = NULL;
+
+/* Initialize KDF file. */
+void
+init_kdf(void)
+{
+ KDFParamsPBKDF2 *params;
+ struct timeval tv;
+ uint64 salt;
+
+ /*
+ * The initialization should not be repeated.
+ */
+ Assert(KDFParams == NULL);
+
+ KDFParams = palloc0(KDF_PARAMS_FILE_SIZE);
+ KDFParams->function = KDF_OPENSSL_PKCS5_PBKDF2_HMAC_SHA;
+ params = &KDFParams->data.pbkdf2;
+
+ /*
+ * Currently we derive the salt in the same way as system identifier,
+ * however these two values are not supposed to match. XXX Is it worth the
+ * effort if initdb derives the system identifier, passes it to this
+ * function and also sends it to the bootstrap process? Not sure.
+ */
+ gettimeofday(&tv, NULL);
+ salt = ((uint64) tv.tv_sec) << 32;
+ salt |= ((uint64) tv.tv_usec) << 12;
+ salt |= getpid() & 0xFFF;
+
+ memcpy(params->salt, &salt, sizeof(uint64));
+ params->niter = ENCRYPTION_KDF_NITER;
+}
+
+/*
+ * Write KDFParamsData to file.
+ */
+void
+write_kdf_file(char *dir)
+{
+ char path[MAXPGPATH];
+ int fd;
+
+ Assert(KDFParams != NULL);
+
+ /* Account for both file separator and terminating NULL character. */
+ if ((strlen(dir) + 1 + strlen(KDF_PARAMS_FILE) + 1) > MAXPGPATH)
+ {
+ pg_log_fatal("KDF directory is too long");
+ exit(EXIT_FAILURE);
+ }
+
+ snprintf(path, MAXPGPATH, "%s/%s", dir, KDF_PARAMS_FILE);
+
+ /* Contents are protected with a CRC */
+ INIT_CRC32C(KDFParams->crc);
+ COMP_CRC32C(KDFParams->crc,
+ (char *) KDFParams,
+ offsetof(KDFParamsData, crc));
+ FIN_CRC32C(KDFParams->crc);
+
+ fd = open(path, O_WRONLY | O_CREAT | PG_BINARY,
+ pg_file_create_mode);
+ if (fd < 0)
+ {
+ pg_log_fatal("could not create key derivation file \"%s\": %m", path);
+ exit(EXIT_FAILURE);
+ }
+
+ if (write(fd, KDFParams, KDF_PARAMS_FILE_SIZE) != KDF_PARAMS_FILE_SIZE)
+ {
+ /* if write didn't set errno, assume problem is no disk space */
+ if (errno == 0)
+ errno = ENOSPC;
+ pg_log_fatal("could not write to key derivation file \"%s\": %m",
+ path);
+ exit(EXIT_FAILURE);
+ }
+
+ if (close(fd))
+ {
+ pg_log_fatal("could not close key setup file: %m");
+ exit(EXIT_FAILURE);
+ }
+}
+
+/*
+ * Read KDFParamsData from file and store it in local memory.
+ *
+ * If dir is NULL, assume we're in the data directory.
+ *
+ * postmaster should call the function early enough for any other process to
+ * inherit valid pointer to the data.
+ */
+void
+read_kdf_file(char *dir)
+{
+ pg_crc32c crc;
+ char path[MAXPGPATH];
+ int fd;
+
+ /* Account for both file separator and terminating NULL character. */
+ if ((strlen(dir) + 1 + strlen(KDF_PARAMS_FILE) + 1) > MAXPGPATH)
+ {
+ pg_log_fatal("KDF directory is too long");
+ exit(EXIT_FAILURE);
+ }
+
+ snprintf(path, MAXPGPATH, "%s/%s", dir, KDF_PARAMS_FILE);
+
+ KDFParams = palloc0(KDF_PARAMS_FILE_SIZE);
+ fd = open(path, O_RDONLY | PG_BINARY, S_IRUSR);
+
+ if (fd < 0)
+ {
+ pg_log_fatal("could not open key setup file \"%s\": %m", path);
+ exit(EXIT_FAILURE);
+ }
+
+ if (read(fd, KDFParams, sizeof(KDFParamsData)) != sizeof(KDFParamsData))
+ {
+ pg_log_fatal("could not read from key setup file \"%s\": %m", path);
+ exit(EXIT_FAILURE);
+ }
+
+ close(fd);
+
+ /* Now check the CRC. */
+ INIT_CRC32C(crc);
+ COMP_CRC32C(crc,
+ (char *) KDFParams,
+ offsetof(KDFParamsData, crc));
+ FIN_CRC32C(crc);
+
+ if (!EQ_CRC32C(crc, KDFParams->crc))
+ {
+ pg_log_fatal("incorrect checksum in key setup file \"%s\"", path);
+ exit(EXIT_FAILURE);
+ }
+
+
+ if (KDFParams->function != KDF_OPENSSL_PKCS5_PBKDF2_HMAC_SHA)
+ {
+ pg_log_fatal("unsupported KDF function");
+ exit(EXIT_FAILURE);
+ }
+}
+
+/*
+ * Run the key derivation function and initialize encryption_key variable.
+ */
+void
+derive_key_from_password(unsigned char *encryption_key, const char *password,
+ int len)
+{
+ KDFParamsPBKDF2 *params;
+ int rc;
+
+ params = &KDFParams->data.pbkdf2;
+ rc = PKCS5_PBKDF2_HMAC(password,
+ len,
+ params->salt,
+ ENCRYPTION_KDF_SALT_LEN,
+ params->niter,
+ EVP_sha1(),
+ ENCRYPTION_KEY_LENGTH,
+ encryption_key);
+
+ if (rc != 1)
+ {
+ pg_log_fatal("failed to derive key from password");
+ exit(EXIT_FAILURE);
+ }
+}
+
/*
* Run the command that is supposed to generate encryption key and store it
- * where encryption_key points to.
+ * where encryption_key points to. If valid string is passed for data_dir,
+ * it's used to replace '%D' pattern in the command.
*/
void
-run_encryption_key_command(unsigned char *encryption_key)
+run_encryption_key_command(unsigned char *encryption_key, char *data_dir)
{
FILE *fp;
+ char cmd[MAXPGPATH];
+ char *sp, *dp, *endp;
char *buf;
int read_len, i, c;
Assert(encryption_key_command != NULL &&
strlen(encryption_key_command) > 0);
- fp = popen(encryption_key_command, "r");
+ /*
+ * Replace %D pattern in the command with the actual data directory path.
+ */
+ dp = cmd;
+ endp = cmd + MAXPGPATH - 1;
+ *endp = '\0';
+ for (sp = encryption_key_command; *sp; sp++)
+ {
+ if (*sp == '%')
+ {
+ if (sp[1] == 'D')
+ {
+ if (data_dir == NULL)
+ {
+ pg_log_fatal("data directory is not known, %%D pattern cannot be replaced");
+ exit(EXIT_FAILURE);
+ }
+
+ sp++;
+ strlcpy(dp, data_dir, endp - dp);
+ make_native_path(dp);
+ dp += strlen(dp);
+ }
+ else if (dp < endp)
+ *dp++ = *sp;
+ else
+ break;
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *sp;
+ else
+ break;
+ }
+ }
+ *dp = '\0';
+
+ pg_log_debug("executing encryption key command \"%s\"", cmd);
+
+ fp = popen(cmd, "r");
if (fp == NULL)
{
- pg_log_fatal("Failed to execute \"%s\"", encryption_key_command);
+ pg_log_fatal("Failed to execute \"%s\"", cmd);
exit(EXIT_FAILURE);
}
diff --git a/src/include/fe_utils/encryption.h b/src/include/fe_utils/encryption.h
index 7307557ed6..da302fe494 100644
--- a/src/include/fe_utils/encryption.h
+++ b/src/include/fe_utils/encryption.h
@@ -15,6 +15,12 @@
/* Executable to retrieve the encryption key. */
extern char *encryption_key_command;
-extern void run_encryption_key_command(unsigned char *encryption_key);
+extern void init_kdf(void);
+extern void write_kdf_file(char *dir);
+extern void read_kdf_file(char *dir);
+extern void derive_key_from_password(unsigned char *encryption_key,
+ const char *password, int len);
+extern void run_encryption_key_command(unsigned char *encryption_key,
+ char *data_dir);
extern bool send_key_to_postmaster(const char *host, const char *port,
const unsigned char *encryption_Key);
diff --git a/src/include/storage/encryption.h b/src/include/storage/encryption.h
index 4f7b96d4f3..72bcae0972 100644
--- a/src/include/storage/encryption.h
+++ b/src/include/storage/encryption.h
@@ -57,6 +57,14 @@ typedef enum CipherKind
PG_CIPHER_AES_BLOCK_CBC_256_STREAM_CTR_256
} CipherKind;
+/*
+ * TODO Tune these values.
+ */
+#define ENCRYPTION_PWD_MIN_LENGTH 8
+#define ENCRYPTION_PWD_MAX_LENGTH 16
+#define ENCRYPTION_KDF_NITER 1048576
+#define ENCRYPTION_KDF_SALT_LEN sizeof(uint64)
+
/* Key to encrypt / decrypt data. */
extern unsigned char encryption_key[];
--
2.13.7
--=-=-=
Content-Type: text/x-diff
Content-Disposition: attachment;
filename=v04-0007-Refactoring-patch.-This-only-moves-some-code-to-XLog.patch
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v6 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 39ebcfec32..4c86f7b448 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -110,16 +93,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -134,7 +113,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2380,7 +2359,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3980,7 +3959,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4023,24 +4002,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4054,7 +4035,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4064,7 +4045,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4075,7 +4056,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4089,7 +4070,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4118,10 +4099,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4221,80 +4202,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4314,17 +4224,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4340,38 +4257,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will usually be able to pick one of the first few items, which will
+ * generally be relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4397,7 +4314,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4443,7 +4360,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4461,7 +4378,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4704,11 +4621,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4726,18 +4643,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--YZ5djTAD1cGYuMQK
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v6-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..95b8f69ecf 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--gBBFr7Ir9EOA20Yy
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..95b8f69ecf 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--gBBFr7Ir9EOA20Yy
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..ea17ca4559 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--CE+1k2dSO48ffgeK
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..95b8f69ecf 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--gBBFr7Ir9EOA20Yy
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 198 ++++++++-------------------
1 file changed, 58 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..b30b49702d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,25 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ /* return opposite of qsort comparator for max-heap */
+ return -TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4262,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4319,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4365,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4383,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4626,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4648,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--y0ulUmNC+osPPQO6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 198 ++++++++-------------------
1 file changed, 58 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..b30b49702d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,25 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ /* return opposite of qsort comparator for max-heap */
+ return -TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4262,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4319,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4365,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4383,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4626,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4648,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--y0ulUmNC+osPPQO6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..ea17ca4559 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--CE+1k2dSO48ffgeK
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..95b8f69ecf 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--gBBFr7Ir9EOA20Yy
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v8-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 198 ++++++++-------------------
1 file changed, 58 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..b30b49702d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,25 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ /* return opposite of qsort comparator for max-heap */
+ return -TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4262,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4319,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4365,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4383,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4626,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4648,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--y0ulUmNC+osPPQO6
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v10-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue.
@ 2023-07-20 17:19 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-07-20 17:19 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
Suggested-by: Tom Lane
Tested-by: Pierre Ducroquet
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/3612876.1689443232%40sss.pgh.pa.us
---
src/bin/pg_dump/pg_backup_archiver.c | 197 ++++++++-------------------
1 file changed, 57 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 4d83381d84..ea17ca4559 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -111,16 +94,12 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -135,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2384,7 +2363,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3984,7 +3963,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4027,24 +4006,26 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4058,7 +4039,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4068,7 +4049,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4079,7 +4060,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4093,7 +4074,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4122,10 +4103,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4225,80 +4206,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4318,17 +4228,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(void *p1, void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(&p1, &p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4344,38 +4261,38 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te = (TocEntry *) binaryheap_get_node(ready_heap, i);
bool conflicts = false;
/*
@@ -4401,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_node(ready_heap, i);
return te;
}
@@ -4447,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4465,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4708,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4730,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, otherte);
}
}
}
--
2.25.1
--CE+1k2dSO48ffgeK
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v9-0004-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue.
@ 2023-09-04 22:35 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-09-04 22:35 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
---
src/bin/pg_dump/pg_backup_archiver.c | 201 ++++++++-------------------
1 file changed, 61 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 39ebcfec32..ff7349537e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -110,16 +93,13 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(const void *p1, const void *p2,
+ void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -134,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2380,7 +2360,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3980,7 +3960,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4023,24 +4003,27 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ sizeof(TocEntry *),
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4054,7 +4037,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4064,7 +4047,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4075,7 +4058,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4089,7 +4072,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4118,10 +4101,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4221,80 +4204,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4314,17 +4226,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(const void *p1, const void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4340,40 +4259,42 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te;
bool conflicts = false;
+ binaryheap_nth(ready_heap, i, &te);
+
/*
* Check to see if the item would need exclusive lock on something
* that a currently running item also needs lock on, or vice versa. If
@@ -4397,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_nth(ready_heap, i);
return te;
}
@@ -4443,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4461,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4704,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4726,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &otherte);
}
}
}
--
2.25.1
--DocE+STaALJfprDB
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v7-0005-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue.
@ 2023-09-04 22:35 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-09-04 22:35 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
---
src/bin/pg_dump/pg_backup_archiver.c | 201 ++++++++-------------------
1 file changed, 61 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 39ebcfec32..ff7349537e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -110,16 +93,13 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(const void *p1, const void *p2,
+ void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -134,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2380,7 +2360,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3980,7 +3960,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4023,24 +4003,27 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ sizeof(TocEntry *),
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4054,7 +4037,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4064,7 +4047,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4075,7 +4058,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4089,7 +4072,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4118,10 +4101,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4221,80 +4204,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4314,17 +4226,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(const void *p1, const void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4340,40 +4259,42 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te;
bool conflicts = false;
+ binaryheap_nth(ready_heap, i, &te);
+
/*
* Check to see if the item would need exclusive lock on something
* that a currently running item also needs lock on, or vice versa. If
@@ -4397,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_nth(ready_heap, i);
return te;
}
@@ -4443,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4461,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4704,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4726,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &otherte);
}
}
}
--
2.25.1
--DocE+STaALJfprDB
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v7-0005-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue.
@ 2023-09-04 22:35 Nathan Bossart <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Nathan Bossart @ 2023-09-04 22:35 UTC (permalink / raw)
Presently, we spend a lot of time sorting this list so that we pick
the largest items first. With many tables, this sorting can become
a significant bottleneck. There are a couple of reports from the
field about this, and it is easily reproducible, so this is not a
hypothetical issue.
This commit improves the performance of pg_restore with many tables
by converting its ready_list to a priority queue, i.e., a binary
heap. We will first try to run the highest priority item, but if
it cannot be chosen due to the lock heuristic, we'll do a
sequential scan through the heap nodes until we find one that is
runnable. This means that we might end up picking an item with
much lower priority, but since we expect that we'll typically be
able to choose one of the first few nodes, we should usually pick
an item with a relatively high priority.
On my machine, a basic test with 100,000 tables takes 11.5 minutes
without this patch and 1.5 minutes with it. Pierre Ducroquet
claims to see a speedup from 30 minutes to 23 minutes for a
real-world dump of over 50,000 tables.
---
src/bin/pg_dump/pg_backup_archiver.c | 201 ++++++++-------------------
1 file changed, 61 insertions(+), 140 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 39ebcfec32..ff7349537e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -34,6 +34,7 @@
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
+#include "lib/binaryheap.h"
#include "lib/stringinfo.h"
#include "libpq/libpq-fs.h"
#include "parallel.h"
@@ -44,24 +45,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/*
- * State for tracking TocEntrys that are ready to process during a parallel
- * restore. (This used to be a list, and we still call it that, though now
- * it's really an array so that we can apply qsort to it.)
- *
- * tes[] is sized large enough that we can't overrun it.
- * The valid entries are indexed first_te .. last_te inclusive.
- * We periodically sort the array to bring larger-by-dataLength entries to
- * the front; "sorted" is true if the valid entries are known sorted.
- */
-typedef struct _parallelReadyList
-{
- TocEntry **tes; /* Ready-to-dump TocEntrys */
- int first_te; /* index of first valid entry in tes[] */
- int last_te; /* index of last valid entry in tes[] */
- bool sorted; /* are valid entries currently sorted? */
-} ParallelReadyList;
-
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -110,16 +93,13 @@ static void restore_toc_entries_postfork(ArchiveHandle *AH,
static void pending_list_header_init(TocEntry *l);
static void pending_list_append(TocEntry *l, TocEntry *te);
static void pending_list_remove(TocEntry *te);
-static void ready_list_init(ParallelReadyList *ready_list, int tocCount);
-static void ready_list_free(ParallelReadyList *ready_list);
-static void ready_list_insert(ParallelReadyList *ready_list, TocEntry *te);
-static void ready_list_remove(ParallelReadyList *ready_list, int i);
-static void ready_list_sort(ParallelReadyList *ready_list);
-static int TocEntrySizeCompare(const void *p1, const void *p2);
-static void move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+static int TocEntrySizeCompareQsort(const void *p1, const void *p2);
+static int TocEntrySizeCompareBinaryheap(const void *p1, const void *p2,
+ void *arg);
+static void move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass);
-static TocEntry *pop_next_work_item(ParallelReadyList *ready_list,
+static TocEntry *pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate);
static void mark_dump_job_done(ArchiveHandle *AH,
TocEntry *te,
@@ -134,7 +114,7 @@ static bool has_lock_conflicts(TocEntry *te1, TocEntry *te2);
static void repoint_table_dependencies(ArchiveHandle *AH);
static void identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te);
static void reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list);
+ binaryheap *ready_heap);
static void mark_create_done(ArchiveHandle *AH, TocEntry *te);
static void inhibit_data_for_failed_table(ArchiveHandle *AH, TocEntry *te);
@@ -2380,7 +2360,7 @@ WriteDataChunks(ArchiveHandle *AH, ParallelState *pstate)
}
if (ntes > 1)
- qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompare);
+ qsort(tes, ntes, sizeof(TocEntry *), TocEntrySizeCompareQsort);
for (int i = 0; i < ntes; i++)
DispatchJobForTocEntry(AH, pstate, tes[i], ACT_DUMP,
@@ -3980,7 +3960,7 @@ restore_toc_entries_prefork(ArchiveHandle *AH, TocEntry *pending_list)
(void) restore_toc_entry(AH, next_work_item, false);
- /* Reduce dependencies, but don't move anything to ready_list */
+ /* Reduce dependencies, but don't move anything to ready_heap */
reduce_dependencies(AH, next_work_item, NULL);
}
else
@@ -4023,24 +4003,27 @@ static void
restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
TocEntry *pending_list)
{
- ParallelReadyList ready_list;
+ binaryheap *ready_heap;
TocEntry *next_work_item;
pg_log_debug("entering restore_toc_entries_parallel");
- /* Set up ready_list with enough room for all known TocEntrys */
- ready_list_init(&ready_list, AH->tocCount);
+ /* Set up ready_heap with enough room for all known TocEntrys */
+ ready_heap = binaryheap_allocate(AH->tocCount,
+ sizeof(TocEntry *),
+ TocEntrySizeCompareBinaryheap,
+ NULL);
/*
* The pending_list contains all items that we need to restore. Move all
- * items that are available to process immediately into the ready_list.
+ * items that are available to process immediately into the ready_heap.
* After this setup, the pending list is everything that needs to be done
- * but is blocked by one or more dependencies, while the ready list
+ * but is blocked by one or more dependencies, while the ready heap
* contains items that have no remaining dependencies and are OK to
* process in the current restore pass.
*/
AH->restorePass = RESTORE_PASS_MAIN;
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/*
* main parent loop
@@ -4054,7 +4037,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
for (;;)
{
/* Look for an item ready to be dispatched to a worker */
- next_work_item = pop_next_work_item(&ready_list, pstate);
+ next_work_item = pop_next_work_item(ready_heap, pstate);
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
@@ -4064,7 +4047,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item->dumpId,
next_work_item->desc, next_work_item->tag);
/* Update its dependencies as though we'd completed it */
- reduce_dependencies(AH, next_work_item, &ready_list);
+ reduce_dependencies(AH, next_work_item, ready_heap);
/* Loop around to see if anything else can be dispatched */
continue;
}
@@ -4075,7 +4058,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Dispatch to some worker */
DispatchJobForTocEntry(AH, pstate, next_work_item, ACT_RESTORE,
- mark_restore_job_done, &ready_list);
+ mark_restore_job_done, ready_heap);
}
else if (IsEveryWorkerIdle(pstate))
{
@@ -4089,7 +4072,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
/* Advance to next restore pass */
AH->restorePass++;
/* That probably allows some stuff to be made ready */
- move_to_ready_list(pending_list, &ready_list, AH->restorePass);
+ move_to_ready_heap(pending_list, ready_heap, AH->restorePass);
/* Loop around to see if anything's now ready */
continue;
}
@@ -4118,10 +4101,10 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
next_work_item ? WFW_ONE_IDLE : WFW_GOT_STATUS);
}
- /* There should now be nothing in ready_list. */
- Assert(ready_list.first_te > ready_list.last_te);
+ /* There should now be nothing in ready_heap. */
+ Assert(binaryheap_empty(ready_heap));
- ready_list_free(&ready_list);
+ binaryheap_free(ready_heap);
pg_log_info("finished main parallel loop");
}
@@ -4221,80 +4204,9 @@ pending_list_remove(TocEntry *te)
}
-/*
- * Initialize the ready_list with enough room for up to tocCount entries.
- */
-static void
-ready_list_init(ParallelReadyList *ready_list, int tocCount)
-{
- ready_list->tes = (TocEntry **)
- pg_malloc(tocCount * sizeof(TocEntry *));
- ready_list->first_te = 0;
- ready_list->last_te = -1;
- ready_list->sorted = false;
-}
-
-/*
- * Free storage for a ready_list.
- */
-static void
-ready_list_free(ParallelReadyList *ready_list)
-{
- pg_free(ready_list->tes);
-}
-
-/* Add te to the ready_list */
-static void
-ready_list_insert(ParallelReadyList *ready_list, TocEntry *te)
-{
- ready_list->tes[++ready_list->last_te] = te;
- /* List is (probably) not sorted anymore. */
- ready_list->sorted = false;
-}
-
-/* Remove the i'th entry in the ready_list */
-static void
-ready_list_remove(ParallelReadyList *ready_list, int i)
-{
- int f = ready_list->first_te;
-
- Assert(i >= f && i <= ready_list->last_te);
-
- /*
- * In the typical case where the item to be removed is the first ready
- * entry, we need only increment first_te to remove it. Otherwise, move
- * the entries before it to compact the list. (This preserves sortedness,
- * if any.) We could alternatively move the entries after i, but there
- * are typically many more of those.
- */
- if (i > f)
- {
- TocEntry **first_te_ptr = &ready_list->tes[f];
-
- memmove(first_te_ptr + 1, first_te_ptr, (i - f) * sizeof(TocEntry *));
- }
- ready_list->first_te++;
-}
-
-/* Sort the ready_list into the desired order */
-static void
-ready_list_sort(ParallelReadyList *ready_list)
-{
- if (!ready_list->sorted)
- {
- int n = ready_list->last_te - ready_list->first_te + 1;
-
- if (n > 1)
- qsort(ready_list->tes + ready_list->first_te, n,
- sizeof(TocEntry *),
- TocEntrySizeCompare);
- ready_list->sorted = true;
- }
-}
-
/* qsort comparator for sorting TocEntries by dataLength */
static int
-TocEntrySizeCompare(const void *p1, const void *p2)
+TocEntrySizeCompareQsort(const void *p1, const void *p2)
{
const TocEntry *te1 = *(const TocEntry *const *) p1;
const TocEntry *te2 = *(const TocEntry *const *) p2;
@@ -4314,17 +4226,24 @@ TocEntrySizeCompare(const void *p1, const void *p2)
return 0;
}
+/* binaryheap comparator for sorting TocEntries by dataLength */
+static int
+TocEntrySizeCompareBinaryheap(const void *p1, const void *p2, void *arg)
+{
+ return TocEntrySizeCompareQsort(p1, p2);
+}
+
/*
- * Move all immediately-ready items from pending_list to ready_list.
+ * Move all immediately-ready items from pending_list to ready_heap.
*
* Items are considered ready if they have no remaining dependencies and
* they belong in the current restore pass. (See also reduce_dependencies,
* which applies the same logic one-at-a-time.)
*/
static void
-move_to_ready_list(TocEntry *pending_list,
- ParallelReadyList *ready_list,
+move_to_ready_heap(TocEntry *pending_list,
+ binaryheap *ready_heap,
RestorePass pass)
{
TocEntry *te;
@@ -4340,40 +4259,42 @@ move_to_ready_list(TocEntry *pending_list,
{
/* Remove it from pending_list ... */
pending_list_remove(te);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, te);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &te);
}
}
}
/*
* Find the next work item (if any) that is capable of being run now,
- * and remove it from the ready_list.
+ * and remove it from the ready_heap.
*
* Returns the item, or NULL if nothing is runnable.
*
* To qualify, the item must have no remaining dependencies
* and no requirements for locks that are incompatible with
- * items currently running. Items in the ready_list are known to have
+ * items currently running. Items in the ready_heap are known to have
* no remaining dependencies, but we have to check for lock conflicts.
*/
static TocEntry *
-pop_next_work_item(ParallelReadyList *ready_list,
+pop_next_work_item(binaryheap *ready_heap,
ParallelState *pstate)
{
/*
- * Sort the ready_list so that we'll tackle larger jobs first.
- */
- ready_list_sort(ready_list);
-
- /*
- * Search the ready_list until we find a suitable item.
+ * Search the ready_heap until we find a suitable item. Note that we do a
+ * sequential scan through the heap nodes, so even though we will first
+ * try to choose the highest-priority item, we might end up picking
+ * something with a much lower priority. However, it is expected that we
+ * will typically be able to pick one of the first few items, which should
+ * usually have a relatively high priority.
*/
- for (int i = ready_list->first_te; i <= ready_list->last_te; i++)
+ for (int i = 0; i < binaryheap_size(ready_heap); i++)
{
- TocEntry *te = ready_list->tes[i];
+ TocEntry *te;
bool conflicts = false;
+ binaryheap_nth(ready_heap, i, &te);
+
/*
* Check to see if the item would need exclusive lock on something
* that a currently running item also needs lock on, or vice versa. If
@@ -4397,7 +4318,7 @@ pop_next_work_item(ParallelReadyList *ready_list,
continue;
/* passed all tests, so this item can run */
- ready_list_remove(ready_list, i);
+ binaryheap_remove_nth(ready_heap, i);
return te;
}
@@ -4443,7 +4364,7 @@ mark_restore_job_done(ArchiveHandle *AH,
int status,
void *callback_data)
{
- ParallelReadyList *ready_list = (ParallelReadyList *) callback_data;
+ binaryheap *ready_heap = (binaryheap *) callback_data;
pg_log_info("finished item %d %s %s",
te->dumpId, te->desc, te->tag);
@@ -4461,7 +4382,7 @@ mark_restore_job_done(ArchiveHandle *AH,
pg_fatal("worker process failed: exit code %d",
status);
- reduce_dependencies(AH, te, ready_list);
+ reduce_dependencies(AH, te, ready_heap);
}
@@ -4704,11 +4625,11 @@ identify_locking_dependencies(ArchiveHandle *AH, TocEntry *te)
/*
* Remove the specified TOC entry from the depCounts of items that depend on
* it, thereby possibly making them ready-to-run. Any pending item that
- * becomes ready should be moved to the ready_list, if that's provided.
+ * becomes ready should be moved to the ready_heap, if that's provided.
*/
static void
reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
- ParallelReadyList *ready_list)
+ binaryheap *ready_heap)
{
int i;
@@ -4726,18 +4647,18 @@ reduce_dependencies(ArchiveHandle *AH, TocEntry *te,
* the current restore pass, and it is currently a member of the
* pending list (that check is needed to prevent double restore in
* some cases where a list-file forces out-of-order restoring).
- * However, if ready_list == NULL then caller doesn't want any list
+ * However, if ready_heap == NULL then caller doesn't want any list
* memberships changed.
*/
if (otherte->depCount == 0 &&
_tocEntryRestorePass(otherte) == AH->restorePass &&
otherte->pending_prev != NULL &&
- ready_list != NULL)
+ ready_heap != NULL)
{
/* Remove it from pending list ... */
pending_list_remove(otherte);
- /* ... and add to ready_list */
- ready_list_insert(ready_list, otherte);
+ /* ... and add to ready_heap */
+ binaryheap_add(ready_heap, &otherte);
}
}
}
--
2.25.1
--DocE+STaALJfprDB
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v7-0005-Remove-open-coded-binary-heap-in-pg_dump_sort.c.patch"
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-05-13 07:20 Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Peter Eisentraut @ 2024-05-13 07:20 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; PostgreSQL Developers <[email protected]>
On 08.04.24 21:29, Daniel Gustafsson wrote:
> Over in [0] I asked whether it would be worthwhile converting all our README
> files to Markdown, and since it wasn't met with pitchforks I figured it would
> be an interesting excercise to see what it would take (my honest gut feeling
> was that it would be way too intrusive). Markdown does brings a few key
> features however so IMHO it's worth attempting to see:
>
> * New developers are very used to reading/writing it
> * Using a defined format ensures some level of consistency
> * Many users and contributors new*as well as* old like reading documentation
> nicely formatted in a browser
> * The documentation now prints really well
> * pandoc et.al can be used to render nice looking PDF's
> * All the same benefits as discussed in [0]
>
> The plan was to follow Grubers original motivation for Markdown closely:
>
> "The idea is that a Markdown-formatted document should be publishable
> as-is, as plain text, without looking like it’s been marked up with
> tags or formatting instructions."
>
> This translates to making the least amount of changes to achieve a) retained
> plain text readability at todays level, b) proper Markdown rendering, not
> looking like text files in a HTML window, and c) absolutly no reflows and
> minimal impact on git blame.
I started looking through this and immediately found a bunch of tiny
problems. (This is probably in part because the READMEs under
src/backend/access/ are some of the more complicated ones, but then they
are also the ones that might benefit most from better rendering.)
One general problem is that original Markdown and GitHub-flavored
Markdown (GFM) are incompatible in some interesting aspects. For
example, the line
A split initially marks the left page with the F_FOLLOW_RIGHT flag.
is rendered by GFM as you'd expect. But original Markdown converts it to
A split initially marks the left page with the F<em>FOLLOW</em>RIGHT
flag.
This kind of problem is pervasive, as you'd expect.
Another incompatibility is that GFM accepts "1)" as a list marker (which
appears to be used often in the READMEs), but original Markdown does
not. This then also affects surrounding formatting.
Also, the READMEs often do not indent lists in a non-ambiguous way. For
example, if you look into src/backend/optimizer/README, section "Join
Tree Construction", there are two list items, but it's not immediately
clear which paragraphs belong to the list and which ones follow the
list. This also interacts with the previous point. The resulting
formatting in GFM is quite misleading.
src/port/README.md is a similar case.
There are also various places where whitespace is used for ad-hoc
formatting. Consider for example in src/backend/access/gin/README
the "category" of the null entry. These are the possible categories:
1 = ordinary null key value extracted from an indexable item
2 = placeholder for zero-key indexable item
3 = placeholder for null indexable item
Placeholder null entries are inserted into the index because otherwise
But this does not preserve the list-like formatting, it just flows it
together.
There is a similar case with the authors list at the end of
src/backend/access/gist/README.md.
src/test/README.md wasn't touched by your patch, but it also needs
adjustments for list formatting.
In summary, I think before we could accept this, we'd need to go through
this with a fine-toothed comb line by line and page by page to make sure
the formatting is still sound. And we'd need to figure out which
Markdown flavor to target.
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-05-15 12:26 Daniel Gustafsson <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 2 replies; 33+ messages in thread
From: Daniel Gustafsson @ 2024-05-15 12:26 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
> On 13 May 2024, at 09:20, Peter Eisentraut <[email protected]> wrote:
> I started looking through this and immediately found a bunch of tiny problems. (This is probably in part because the READMEs under src/backend/access/ are some of the more complicated ones, but then they are also the ones that might benefit most from better rendering.)
Thanks for looking!
> One general problem is that original Markdown and GitHub-flavored Markdown (GFM) are incompatible in some interesting aspects.
That's true, but virtually every implementation of Markdown in practical use
today is incompatible with Original Markdown.
Reading my email I realize I failed to mention the markdown platforms I was
targeting (and thus flavours), and citing Gruber made it even more confusing.
For online reading I verified with Github and VS Code since they have a huge
market presence. For offline work I targeted rendering with pandoc since we
already have a dependency on it in the tree. I don't think targeting the
original Markdown implementation is useful, or even realistic.
Another aspect of platform/flavour was to make the markdown version easy to
maintain for hackers writing content. Requiring the minimum amount of markup
seems like the developer-friendly way here to keep productivity as well as
document quality high.
Most importantly though, I targeted reading the files as plain text without any
rendering. We keep these files in text format close to the code for a reason,
and maintaining readability as text was a north star.
> For example, the line
>
> A split initially marks the left page with the F_FOLLOW_RIGHT flag.
>
> is rendered by GFM as you'd expect. But original Markdown converts it to
>
> A split initially marks the left page with the F<em>FOLLOW</em>RIGHT
> flag.
>
> This kind of problem is pervasive, as you'd expect.
Correct, but I can't imagine that we'd like to wrap every instance of a name
with underscores in backticks like `F_FOLLOW_RIGHT`. There are very few
Markdown implementations which don't support underscores like this (testing
just now on the top online editors and sites providing markdown editing I
failed to find a single one).
> Also, the READMEs often do not indent lists in a non-ambiguous way. For example, if you look into src/backend/optimizer/README, section "Join Tree Construction", there are two list items, but it's not immediately clear which paragraphs belong to the list and which ones follow the list. This also interacts with the previous point. The resulting formatting in GFM is quite misleading.
I agree that the rendered version excacerbates this problem. Writing a bullet
point list where each item spans multiple paragraphs indented the same way as
the paragraphs following the list is not helpful to the reader. In these cases
both the markdown and the text version will be improved by indentation.
> There are also various places where whitespace is used for ad-hoc formatting. Consider for example in src/backend/access/gin/README
>
> the "category" of the null entry. These are the possible categories:
>
> 1 = ordinary null key value extracted from an indexable item
> 2 = placeholder for zero-key indexable item
> 3 = placeholder for null indexable item
>
> Placeholder null entries are inserted into the index because otherwise
>
> But this does not preserve the list-like formatting, it just flows it together.
That's the kind of sublists which need to be found as part of this work, and
the items prefixed with a list identifier. In this case, prefixing each row in
the sublist with '-' yields the correct result.
> src/test/README.md wasn't touched by your patch, but it also needs adjustments for list formatting.
I didn't re-indent that one in order to keep the changes to the absolute
minimum, since I considered the rendered version passable even if not
particularly good. Re-indenting files like this will for sure make the end
result better, as long as the changes keep the text version readability.
> In summary, I think before we could accept this, we'd need to go through this with a fine-toothed comb line by line and page by page to make sure the formatting is still sound.
Absolutely. I've been over every file to ensure they aren't blatantly wrong,
but I didn't want to spend the time if this was immmediately shot down as
something the community don't want to maintain.
> And we'd need to figure out which Markdown flavor to target.
Absolutely, and as I mentioned above, we need to pick based both the final
result (text and rendered) as well as the developer experience for maintaining
this.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-06-28 07:38 Peter Eisentraut <[email protected]>
parent: Daniel Gustafsson <[email protected]>
1 sibling, 2 replies; 33+ messages in thread
From: Peter Eisentraut @ 2024-06-28 07:38 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
On 15.05.24 14:26, Daniel Gustafsson wrote:
> Another aspect of platform/flavour was to make the markdown version easy to
> maintain for hackers writing content. Requiring the minimum amount of markup
> seems like the developer-friendly way here to keep productivity as well as
> document quality high.
>
> Most importantly though, I targeted reading the files as plain text without any
> rendering. We keep these files in text format close to the code for a reason,
> and maintaining readability as text was a north star.
I've been thinking about this some more. I think the most value here
would be to just improve the plain-text formatting, so that there are
consistent list styles, header styles, indentation, some of the
ambiguities cleared up -- much of which your 0001 patch does. You might
as well be targeting markdown-like conventions with this; they are
mostly reasonable.
I tend to think that actually converting all the README files to
README.md could be a net negative for maintainability. Because now you
are requiring everyone who potentially wants to edit those to be aware
of Markdown syntax and manually check the rendering. With things like
DocBook, if you make a mess, you get error messages from the build step.
If you make a mess in Markdown, you have to visually find it yourself.
There are many READMEs that contain nested lists and code snippets and
diagrams and such all mixed together. Getting that right in Markdown
can be quite tricky. I'm also foreseeing related messes of trailing
whitespace, spaces-vs-tab confusion, gitattributes violations, etc. It
can be a lot of effort. It's okay to do this for prominent files like
the top-level one, but I suggest that for the rest we can keep it simple
and just use plain text.
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-06-28 09:56 Jelte Fennema-Nio <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Jelte Fennema-Nio @ 2024-06-28 09:56 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Developers <[email protected]>
On Fri, 28 Jun 2024 at 09:38, Peter Eisentraut <[email protected]> wrote:
> Getting that right in Markdown can be quite tricky.
I agree that in some cases it's tricky. But what's the worst case that
can happen when you get it wrong? It renders weird on github.com.
Luckily there's a "code" button to go to the plain text format[1]. In
all other cases (which I expect will be most) the doc will be easier
to read. Forcing plaintext, just because sometimes we might make a
mistake in the syntax seems like an overcorrection imho. Especially
because these docs are (hopefully) read more often than written.
[1]: https://github.com/postgres/postgres/blob/master/README.md?plain=1
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-06-28 12:37 Tatsuo Ishii <[email protected]>
parent: Daniel Gustafsson <[email protected]>
1 sibling, 0 replies; 33+ messages in thread
From: Tatsuo Ishii @ 2024-06-28 12:37 UTC (permalink / raw)
To: [email protected]; +Cc: [email protected]; [email protected]
> I've been thinking about this some more. I think the most value here
> would be to just improve the plain-text formatting, so that there are
> consistent list styles, header styles, indentation, some of the
> ambiguities cleared up -- much of which your 0001 patch does. You
> might as well be targeting markdown-like conventions with this; they
> are mostly reasonable.
>
> I tend to think that actually converting all the README files to
> README.md could be a net negative for maintainability. Because now
> you are requiring everyone who potentially wants to edit those to be
> aware of Markdown syntax and manually check the rendering. With
> things like DocBook, if you make a mess, you get error messages from
> the build step. If you make a mess in Markdown, you have to visually
> find it yourself. There are many READMEs that contain nested lists
> and code snippets and diagrams and such all mixed together. Getting
> that right in Markdown can be quite tricky. I'm also foreseeing
> related messes of trailing whitespace, spaces-vs-tab confusion,
> gitattributes violations, etc. It can be a lot of effort. It's okay
> to do this for prominent files like the top-level one, but I suggest
> that for the rest we can keep it simple and just use plain text.
Agreed.
Best reagards,
--
Tatsuo Ishii
SRA OSS LLC
English: http://www.sraoss.co.jp/index_en/
Japanese:http://www.sraoss.co.jp
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-06-28 18:40 Peter Eisentraut <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 2 replies; 33+ messages in thread
From: Peter Eisentraut @ 2024-06-28 18:40 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Developers <[email protected]>
On 28.06.24 11:56, Jelte Fennema-Nio wrote:
> On Fri, 28 Jun 2024 at 09:38, Peter Eisentraut <[email protected]> wrote:
>> Getting that right in Markdown can be quite tricky.
>
> I agree that in some cases it's tricky. But what's the worst case that
> can happen when you get it wrong? It renders weird on github.com.
I have my "less" set up so that "less somefile.md" automatically renders
the markdown. That's been pretty useful. But if that now keeps making
a mess out of PostgreSQL's README files, then I'm going to have to keep
fixing things, and I might get really mad. That's the worst that could
happen. ;-)
So I don't agree with "aspirational markdown". If we're going to do it,
then I expect that the files are marked up correctly at all times.
Conversely, what's the best that could happen?
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-06-29 09:24 Jelte Fennema-Nio <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Jelte Fennema-Nio @ 2024-06-29 09:24 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Developers <[email protected]>
On Fri, 28 Jun 2024 at 20:40, Peter Eisentraut <[email protected]> wrote:
> I have my "less" set up so that "less somefile.md" automatically renders
> the markdown. That's been pretty useful. But if that now keeps making
> a mess out of PostgreSQL's README files, then I'm going to have to keep
> fixing things, and I might get really mad. That's the worst that could
> happen. ;-)
Do you have reason to think that this is going to be a bigger issue
for Postgres READMEs than for any other markdown files you encounter?
Because this sounds like a generic problem you'd run into with your
"less" set up, which so far apparently has been small enough that it's
worth the benefit of automatically rendering markdown files.
> So I don't agree with "aspirational markdown". If we're going to do it,
> then I expect that the files are marked up correctly at all times.
I think for at least ~90% of our README files this shouldn't be a
problem. If you have specific ones in mind that contain difficult
markup/diagrams, then maybe we shouldn't convert those.
> Conversely, what's the best that could happen?
That your "less" would automatically render Postgres READMEs nicely.
Which you say has been pretty useful ;-) And maybe even show syntax
highlighting for codeblocks.
P.S. Now I'm wondering what your "less" is.
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-07-01 09:42 Daniel Gustafsson <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 0 replies; 33+ messages in thread
From: Daniel Gustafsson @ 2024-07-01 09:42 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; PostgreSQL Developers <[email protected]>
> On 28 Jun 2024, at 20:40, Peter Eisentraut <[email protected]> wrote:
> If we're going to do it, then I expect that the files are marked up correctly at all times.
I agree with that. I don't think it will be a terribly high bar though since we
were pretty much already writing markdown. We already have pandoc in the meson
toolchain, adding a target to check syntax should be doable.
> Conversely, what's the best that could happen?
One of the main goals of this work was to make sure the documentation renders
nicely on platforms which potential new contributors consider part of the
fabric of writing code. We might not be on Github (and I'm not advocating that
we should) but any new contributor we want to attract is pretty likely to be
using it. The best that can happen is that new contributors find the postgres
code more approachable and get excited about contributing to postgres.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-07-01 10:22 Daniel Gustafsson <[email protected]>
parent: Peter Eisentraut <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Daniel Gustafsson @ 2024-07-01 10:22 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
> On 28 Jun 2024, at 09:38, Peter Eisentraut <[email protected]> wrote:
> I've been thinking about this some more. I think the most value here would be to just improve the plain-text formatting, so that there are consistent list styles, header styles, indentation, some of the ambiguities cleared up -- much of which your 0001 patch does. You might as well be targeting markdown-like conventions with this; they are mostly reasonable.
(I assume you mean 0002). I agree that the increased consistency is worthwhile
even if we don't officially convert to Markdown (ie only do 0002 and not 0001).
> I tend to think that actually converting all the README files to README.md could be a net negative for maintainability. Because now you are requiring everyone who potentially wants to edit those to be aware of Markdown syntax
Fair enough, but we currently expect those editing to be aware of our syntax
which isn't defined at all (leading to the variations this patchset fixes).
I'm not sure whats best for maintainability but I do think the net change is
all that big.
> and manually check the rendering.
That however would be a new requirement, and I can see that being a deal-
breaker for introducing this.
Attached is a v2 which fixes a conflict, if there is no interest in Markdown
I'll drop 0001 and the markdown-specifics from 0002 to instead target increased
consistency.
--
Daniel Gustafsson
Attachments:
[application/octet-stream] v2-0001-Convert-internal-documentation-to-Markdown-rename.patch (30.3K, ../../[email protected]/2-v2-0001-Convert-internal-documentation-to-Markdown-rename.patch)
download | inline diff:
From 58ef6ec089e434bfed58fbbc30515c98afe2b6ee Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <[email protected]>
Date: Fri, 5 Apr 2024 09:25:55 +0200
Subject: [PATCH v2 1/2] Convert internal documentation to Markdown - rename
This renames all README files to use the .md file suffix without
changing any content.
---
contrib/{README => README.md} | 0
contrib/start-scripts/macos/{README => README.md} | 0
src/backend/access/brin/{README => README.md} | 0
src/backend/access/gin/{README => README.md} | 0
src/backend/access/gist/{README => README.md} | 0
src/backend/access/hash/{README => README.md} | 0
src/backend/access/heap/{README.HOT => README.HOT.md} | 0
src/backend/access/heap/{README.tuplock => README.tuplock.md} | 0
src/backend/access/nbtree/{README => README.md} | 0
src/backend/access/rmgrdesc/{README => README.md} | 0
src/backend/access/spgist/{README => README.md} | 0
src/backend/access/transam/{README => README.md} | 0
.../access/transam/{README.parallel => README.parallel.md} | 0
src/backend/executor/{README => README.md} | 0
src/backend/jit/{README => README.md} | 0
src/backend/lib/{README => README.md} | 0
src/backend/libpq/{README.SSL => README.SSL.md} | 0
src/backend/nodes/{README => README.md} | 0
src/backend/optimizer/{README => README.md} | 0
src/backend/optimizer/plan/{README => README.md} | 0
src/backend/parser/{README => README.md} | 0
src/backend/regex/{README => README.md} | 0
src/backend/replication/{README => README.md} | 0
src/backend/snowball/{README => README.md} | 0
.../statistics/{README.dependencies => README.dependencies.md} | 0
src/backend/statistics/{README.mcv => README.mcv.md} | 0
src/backend/statistics/{README => README.md} | 0
src/backend/storage/buffer/{README => README.md} | 0
src/backend/storage/freespace/{README => README.md} | 0
src/backend/storage/lmgr/{README-SSI => README-SSI.md} | 0
src/backend/storage/lmgr/{README.barrier => README.barrier.md} | 0
src/backend/storage/lmgr/{README => README.md} | 0
src/backend/storage/page/{README => README.md} | 0
src/backend/storage/smgr/{README => README.md} | 0
src/backend/utils/fmgr/{README => README.md} | 0
src/backend/utils/mb/{README => README.md} | 0
src/backend/utils/misc/{README => README.md} | 0
src/backend/utils/mmgr/{README => README.md} | 0
src/backend/utils/resowner/{README => README.md} | 0
src/bin/pg_amcheck/{README => README.md} | 0
src/bin/pgevent/{README => README.md} | 0
src/common/unicode/{README => README.md} | 0
src/interfaces/ecpg/{README.dynSQL => README.dynSQL.md} | 0
src/interfaces/ecpg/preproc/{README.parser => README.parser.md} | 0
src/interfaces/ecpg/test/connect/{README => README.md} | 0
src/interfaces/libpq/{README => README.md} | 0
src/pl/plperl/{README => README.md} | 0
src/pl/plpython/expected/{README => README.md} | 0
src/port/{README => README.md} | 0
src/test/{README => README.md} | 0
src/test/authentication/{README => README.md} | 0
src/test/icu/{README => README.md} | 0
src/test/isolation/{README => README.md} | 0
src/test/kerberos/{README => README.md} | 0
src/test/ldap/{README => README.md} | 0
src/test/locale/{README => README.md} | 0
src/test/locale/de_DE.ISO8859-1/{README => README.md} | 0
src/test/locale/gr_GR.ISO8859-7/{README => README.md} | 0
src/test/locale/koi8-to-win1251/{README => README.md} | 0
src/test/mb/{README => README.md} | 0
src/test/modules/{README => README.md} | 0
src/test/modules/dummy_index_am/{README => README.md} | 0
src/test/modules/dummy_seclabel/{README => README.md} | 0
src/test/modules/libpq_pipeline/{README => README.md} | 0
src/test/modules/plsample/{README => README.md} | 0
src/test/modules/spgist_name_ops/{README => README.md} | 0
src/test/modules/test_bloomfilter/{README => README.md} | 0
src/test/modules/test_ddl_deparse/{README => README.md} | 0
src/test/modules/test_ginpostinglist/{README => README.md} | 0
src/test/modules/test_integerset/{README => README.md} | 0
src/test/modules/test_json_parser/{README => README.md} | 0
src/test/modules/test_misc/{README => README.md} | 0
src/test/modules/test_oat_hooks/{README => README.md} | 0
src/test/modules/test_parser/{README => README.md} | 0
src/test/modules/test_pg_dump/{README => README.md} | 0
src/test/modules/test_predtest/{README => README.md} | 0
src/test/modules/test_rbtree/{README => README.md} | 0
src/test/modules/test_regex/{README => README.md} | 0
src/test/modules/test_rls_hooks/{README => README.md} | 0
src/test/modules/test_shm_mq/{README => README.md} | 0
src/test/modules/unsafe_tests/{README => README.md} | 0
src/test/modules/xid_wraparound/{README => README.md} | 0
src/test/perl/{README => README.md} | 0
src/test/recovery/{README => README.md} | 0
src/test/regress/{README => README.md} | 0
src/test/ssl/{README => README.md} | 0
src/test/subscription/{README => README.md} | 0
src/timezone/{README => README.md} | 0
src/timezone/tznames/{README => README.md} | 0
src/tools/ci/{README => README.md} | 0
src/tools/ifaddrs/{README => README.md} | 0
src/tools/pg_bsd_indent/{README => README.md} | 0
src/tools/pginclude/{README => README.md} | 0
src/tools/pgindent/{README => README.md} | 0
src/tutorial/{README => README.md} | 0
95 files changed, 0 insertions(+), 0 deletions(-)
rename contrib/{README => README.md} (100%)
rename contrib/start-scripts/macos/{README => README.md} (100%)
rename src/backend/access/brin/{README => README.md} (100%)
rename src/backend/access/gin/{README => README.md} (100%)
rename src/backend/access/gist/{README => README.md} (100%)
rename src/backend/access/hash/{README => README.md} (100%)
rename src/backend/access/heap/{README.HOT => README.HOT.md} (100%)
rename src/backend/access/heap/{README.tuplock => README.tuplock.md} (100%)
rename src/backend/access/nbtree/{README => README.md} (100%)
rename src/backend/access/rmgrdesc/{README => README.md} (100%)
rename src/backend/access/spgist/{README => README.md} (100%)
rename src/backend/access/transam/{README => README.md} (100%)
rename src/backend/access/transam/{README.parallel => README.parallel.md} (100%)
rename src/backend/executor/{README => README.md} (100%)
rename src/backend/jit/{README => README.md} (100%)
rename src/backend/lib/{README => README.md} (100%)
rename src/backend/libpq/{README.SSL => README.SSL.md} (100%)
rename src/backend/nodes/{README => README.md} (100%)
rename src/backend/optimizer/{README => README.md} (100%)
rename src/backend/optimizer/plan/{README => README.md} (100%)
rename src/backend/parser/{README => README.md} (100%)
rename src/backend/regex/{README => README.md} (100%)
rename src/backend/replication/{README => README.md} (100%)
rename src/backend/snowball/{README => README.md} (100%)
rename src/backend/statistics/{README.dependencies => README.dependencies.md} (100%)
rename src/backend/statistics/{README.mcv => README.mcv.md} (100%)
rename src/backend/statistics/{README => README.md} (100%)
rename src/backend/storage/buffer/{README => README.md} (100%)
rename src/backend/storage/freespace/{README => README.md} (100%)
rename src/backend/storage/lmgr/{README-SSI => README-SSI.md} (100%)
rename src/backend/storage/lmgr/{README.barrier => README.barrier.md} (100%)
rename src/backend/storage/lmgr/{README => README.md} (100%)
rename src/backend/storage/page/{README => README.md} (100%)
rename src/backend/storage/smgr/{README => README.md} (100%)
rename src/backend/utils/fmgr/{README => README.md} (100%)
rename src/backend/utils/mb/{README => README.md} (100%)
rename src/backend/utils/misc/{README => README.md} (100%)
rename src/backend/utils/mmgr/{README => README.md} (100%)
rename src/backend/utils/resowner/{README => README.md} (100%)
rename src/bin/pg_amcheck/{README => README.md} (100%)
rename src/bin/pgevent/{README => README.md} (100%)
rename src/common/unicode/{README => README.md} (100%)
rename src/interfaces/ecpg/{README.dynSQL => README.dynSQL.md} (100%)
rename src/interfaces/ecpg/preproc/{README.parser => README.parser.md} (100%)
rename src/interfaces/ecpg/test/connect/{README => README.md} (100%)
rename src/interfaces/libpq/{README => README.md} (100%)
rename src/pl/plperl/{README => README.md} (100%)
rename src/pl/plpython/expected/{README => README.md} (100%)
rename src/port/{README => README.md} (100%)
rename src/test/{README => README.md} (100%)
rename src/test/authentication/{README => README.md} (100%)
rename src/test/icu/{README => README.md} (100%)
rename src/test/isolation/{README => README.md} (100%)
rename src/test/kerberos/{README => README.md} (100%)
rename src/test/ldap/{README => README.md} (100%)
rename src/test/locale/{README => README.md} (100%)
rename src/test/locale/de_DE.ISO8859-1/{README => README.md} (100%)
rename src/test/locale/gr_GR.ISO8859-7/{README => README.md} (100%)
rename src/test/locale/koi8-to-win1251/{README => README.md} (100%)
rename src/test/mb/{README => README.md} (100%)
rename src/test/modules/{README => README.md} (100%)
rename src/test/modules/dummy_index_am/{README => README.md} (100%)
rename src/test/modules/dummy_seclabel/{README => README.md} (100%)
rename src/test/modules/libpq_pipeline/{README => README.md} (100%)
rename src/test/modules/plsample/{README => README.md} (100%)
rename src/test/modules/spgist_name_ops/{README => README.md} (100%)
rename src/test/modules/test_bloomfilter/{README => README.md} (100%)
rename src/test/modules/test_ddl_deparse/{README => README.md} (100%)
rename src/test/modules/test_ginpostinglist/{README => README.md} (100%)
rename src/test/modules/test_integerset/{README => README.md} (100%)
rename src/test/modules/test_json_parser/{README => README.md} (100%)
rename src/test/modules/test_misc/{README => README.md} (100%)
rename src/test/modules/test_oat_hooks/{README => README.md} (100%)
rename src/test/modules/test_parser/{README => README.md} (100%)
rename src/test/modules/test_pg_dump/{README => README.md} (100%)
rename src/test/modules/test_predtest/{README => README.md} (100%)
rename src/test/modules/test_rbtree/{README => README.md} (100%)
rename src/test/modules/test_regex/{README => README.md} (100%)
rename src/test/modules/test_rls_hooks/{README => README.md} (100%)
rename src/test/modules/test_shm_mq/{README => README.md} (100%)
rename src/test/modules/unsafe_tests/{README => README.md} (100%)
rename src/test/modules/xid_wraparound/{README => README.md} (100%)
rename src/test/perl/{README => README.md} (100%)
rename src/test/recovery/{README => README.md} (100%)
rename src/test/regress/{README => README.md} (100%)
rename src/test/ssl/{README => README.md} (100%)
rename src/test/subscription/{README => README.md} (100%)
rename src/timezone/{README => README.md} (100%)
rename src/timezone/tznames/{README => README.md} (100%)
rename src/tools/ci/{README => README.md} (100%)
rename src/tools/ifaddrs/{README => README.md} (100%)
rename src/tools/pg_bsd_indent/{README => README.md} (100%)
rename src/tools/pginclude/{README => README.md} (100%)
rename src/tools/pgindent/{README => README.md} (100%)
rename src/tutorial/{README => README.md} (100%)
diff --git a/contrib/README b/contrib/README.md
similarity index 100%
rename from contrib/README
rename to contrib/README.md
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README.md
similarity index 100%
rename from contrib/start-scripts/macos/README
rename to contrib/start-scripts/macos/README.md
diff --git a/src/backend/access/brin/README b/src/backend/access/brin/README.md
similarity index 100%
rename from src/backend/access/brin/README
rename to src/backend/access/brin/README.md
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README.md
similarity index 100%
rename from src/backend/access/gin/README
rename to src/backend/access/gin/README.md
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README.md
similarity index 100%
rename from src/backend/access/gist/README
rename to src/backend/access/gist/README.md
diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README.md
similarity index 100%
rename from src/backend/access/hash/README
rename to src/backend/access/hash/README.md
diff --git a/src/backend/access/heap/README.HOT b/src/backend/access/heap/README.HOT.md
similarity index 100%
rename from src/backend/access/heap/README.HOT
rename to src/backend/access/heap/README.HOT.md
diff --git a/src/backend/access/heap/README.tuplock b/src/backend/access/heap/README.tuplock.md
similarity index 100%
rename from src/backend/access/heap/README.tuplock
rename to src/backend/access/heap/README.tuplock.md
diff --git a/src/backend/access/nbtree/README b/src/backend/access/nbtree/README.md
similarity index 100%
rename from src/backend/access/nbtree/README
rename to src/backend/access/nbtree/README.md
diff --git a/src/backend/access/rmgrdesc/README b/src/backend/access/rmgrdesc/README.md
similarity index 100%
rename from src/backend/access/rmgrdesc/README
rename to src/backend/access/rmgrdesc/README.md
diff --git a/src/backend/access/spgist/README b/src/backend/access/spgist/README.md
similarity index 100%
rename from src/backend/access/spgist/README
rename to src/backend/access/spgist/README.md
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README.md
similarity index 100%
rename from src/backend/access/transam/README
rename to src/backend/access/transam/README.md
diff --git a/src/backend/access/transam/README.parallel b/src/backend/access/transam/README.parallel.md
similarity index 100%
rename from src/backend/access/transam/README.parallel
rename to src/backend/access/transam/README.parallel.md
diff --git a/src/backend/executor/README b/src/backend/executor/README.md
similarity index 100%
rename from src/backend/executor/README
rename to src/backend/executor/README.md
diff --git a/src/backend/jit/README b/src/backend/jit/README.md
similarity index 100%
rename from src/backend/jit/README
rename to src/backend/jit/README.md
diff --git a/src/backend/lib/README b/src/backend/lib/README.md
similarity index 100%
rename from src/backend/lib/README
rename to src/backend/lib/README.md
diff --git a/src/backend/libpq/README.SSL b/src/backend/libpq/README.SSL.md
similarity index 100%
rename from src/backend/libpq/README.SSL
rename to src/backend/libpq/README.SSL.md
diff --git a/src/backend/nodes/README b/src/backend/nodes/README.md
similarity index 100%
rename from src/backend/nodes/README
rename to src/backend/nodes/README.md
diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README.md
similarity index 100%
rename from src/backend/optimizer/README
rename to src/backend/optimizer/README.md
diff --git a/src/backend/optimizer/plan/README b/src/backend/optimizer/plan/README.md
similarity index 100%
rename from src/backend/optimizer/plan/README
rename to src/backend/optimizer/plan/README.md
diff --git a/src/backend/parser/README b/src/backend/parser/README.md
similarity index 100%
rename from src/backend/parser/README
rename to src/backend/parser/README.md
diff --git a/src/backend/regex/README b/src/backend/regex/README.md
similarity index 100%
rename from src/backend/regex/README
rename to src/backend/regex/README.md
diff --git a/src/backend/replication/README b/src/backend/replication/README.md
similarity index 100%
rename from src/backend/replication/README
rename to src/backend/replication/README.md
diff --git a/src/backend/snowball/README b/src/backend/snowball/README.md
similarity index 100%
rename from src/backend/snowball/README
rename to src/backend/snowball/README.md
diff --git a/src/backend/statistics/README.dependencies b/src/backend/statistics/README.dependencies.md
similarity index 100%
rename from src/backend/statistics/README.dependencies
rename to src/backend/statistics/README.dependencies.md
diff --git a/src/backend/statistics/README.mcv b/src/backend/statistics/README.mcv.md
similarity index 100%
rename from src/backend/statistics/README.mcv
rename to src/backend/statistics/README.mcv.md
diff --git a/src/backend/statistics/README b/src/backend/statistics/README.md
similarity index 100%
rename from src/backend/statistics/README
rename to src/backend/statistics/README.md
diff --git a/src/backend/storage/buffer/README b/src/backend/storage/buffer/README.md
similarity index 100%
rename from src/backend/storage/buffer/README
rename to src/backend/storage/buffer/README.md
diff --git a/src/backend/storage/freespace/README b/src/backend/storage/freespace/README.md
similarity index 100%
rename from src/backend/storage/freespace/README
rename to src/backend/storage/freespace/README.md
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI.md
similarity index 100%
rename from src/backend/storage/lmgr/README-SSI
rename to src/backend/storage/lmgr/README-SSI.md
diff --git a/src/backend/storage/lmgr/README.barrier b/src/backend/storage/lmgr/README.barrier.md
similarity index 100%
rename from src/backend/storage/lmgr/README.barrier
rename to src/backend/storage/lmgr/README.barrier.md
diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README.md
similarity index 100%
rename from src/backend/storage/lmgr/README
rename to src/backend/storage/lmgr/README.md
diff --git a/src/backend/storage/page/README b/src/backend/storage/page/README.md
similarity index 100%
rename from src/backend/storage/page/README
rename to src/backend/storage/page/README.md
diff --git a/src/backend/storage/smgr/README b/src/backend/storage/smgr/README.md
similarity index 100%
rename from src/backend/storage/smgr/README
rename to src/backend/storage/smgr/README.md
diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README.md
similarity index 100%
rename from src/backend/utils/fmgr/README
rename to src/backend/utils/fmgr/README.md
diff --git a/src/backend/utils/mb/README b/src/backend/utils/mb/README.md
similarity index 100%
rename from src/backend/utils/mb/README
rename to src/backend/utils/mb/README.md
diff --git a/src/backend/utils/misc/README b/src/backend/utils/misc/README.md
similarity index 100%
rename from src/backend/utils/misc/README
rename to src/backend/utils/misc/README.md
diff --git a/src/backend/utils/mmgr/README b/src/backend/utils/mmgr/README.md
similarity index 100%
rename from src/backend/utils/mmgr/README
rename to src/backend/utils/mmgr/README.md
diff --git a/src/backend/utils/resowner/README b/src/backend/utils/resowner/README.md
similarity index 100%
rename from src/backend/utils/resowner/README
rename to src/backend/utils/resowner/README.md
diff --git a/src/bin/pg_amcheck/README b/src/bin/pg_amcheck/README.md
similarity index 100%
rename from src/bin/pg_amcheck/README
rename to src/bin/pg_amcheck/README.md
diff --git a/src/bin/pgevent/README b/src/bin/pgevent/README.md
similarity index 100%
rename from src/bin/pgevent/README
rename to src/bin/pgevent/README.md
diff --git a/src/common/unicode/README b/src/common/unicode/README.md
similarity index 100%
rename from src/common/unicode/README
rename to src/common/unicode/README.md
diff --git a/src/interfaces/ecpg/README.dynSQL b/src/interfaces/ecpg/README.dynSQL.md
similarity index 100%
rename from src/interfaces/ecpg/README.dynSQL
rename to src/interfaces/ecpg/README.dynSQL.md
diff --git a/src/interfaces/ecpg/preproc/README.parser b/src/interfaces/ecpg/preproc/README.parser.md
similarity index 100%
rename from src/interfaces/ecpg/preproc/README.parser
rename to src/interfaces/ecpg/preproc/README.parser.md
diff --git a/src/interfaces/ecpg/test/connect/README b/src/interfaces/ecpg/test/connect/README.md
similarity index 100%
rename from src/interfaces/ecpg/test/connect/README
rename to src/interfaces/ecpg/test/connect/README.md
diff --git a/src/interfaces/libpq/README b/src/interfaces/libpq/README.md
similarity index 100%
rename from src/interfaces/libpq/README
rename to src/interfaces/libpq/README.md
diff --git a/src/pl/plperl/README b/src/pl/plperl/README.md
similarity index 100%
rename from src/pl/plperl/README
rename to src/pl/plperl/README.md
diff --git a/src/pl/plpython/expected/README b/src/pl/plpython/expected/README.md
similarity index 100%
rename from src/pl/plpython/expected/README
rename to src/pl/plpython/expected/README.md
diff --git a/src/port/README b/src/port/README.md
similarity index 100%
rename from src/port/README
rename to src/port/README.md
diff --git a/src/test/README b/src/test/README.md
similarity index 100%
rename from src/test/README
rename to src/test/README.md
diff --git a/src/test/authentication/README b/src/test/authentication/README.md
similarity index 100%
rename from src/test/authentication/README
rename to src/test/authentication/README.md
diff --git a/src/test/icu/README b/src/test/icu/README.md
similarity index 100%
rename from src/test/icu/README
rename to src/test/icu/README.md
diff --git a/src/test/isolation/README b/src/test/isolation/README.md
similarity index 100%
rename from src/test/isolation/README
rename to src/test/isolation/README.md
diff --git a/src/test/kerberos/README b/src/test/kerberos/README.md
similarity index 100%
rename from src/test/kerberos/README
rename to src/test/kerberos/README.md
diff --git a/src/test/ldap/README b/src/test/ldap/README.md
similarity index 100%
rename from src/test/ldap/README
rename to src/test/ldap/README.md
diff --git a/src/test/locale/README b/src/test/locale/README.md
similarity index 100%
rename from src/test/locale/README
rename to src/test/locale/README.md
diff --git a/src/test/locale/de_DE.ISO8859-1/README b/src/test/locale/de_DE.ISO8859-1/README.md
similarity index 100%
rename from src/test/locale/de_DE.ISO8859-1/README
rename to src/test/locale/de_DE.ISO8859-1/README.md
diff --git a/src/test/locale/gr_GR.ISO8859-7/README b/src/test/locale/gr_GR.ISO8859-7/README.md
similarity index 100%
rename from src/test/locale/gr_GR.ISO8859-7/README
rename to src/test/locale/gr_GR.ISO8859-7/README.md
diff --git a/src/test/locale/koi8-to-win1251/README b/src/test/locale/koi8-to-win1251/README.md
similarity index 100%
rename from src/test/locale/koi8-to-win1251/README
rename to src/test/locale/koi8-to-win1251/README.md
diff --git a/src/test/mb/README b/src/test/mb/README.md
similarity index 100%
rename from src/test/mb/README
rename to src/test/mb/README.md
diff --git a/src/test/modules/README b/src/test/modules/README.md
similarity index 100%
rename from src/test/modules/README
rename to src/test/modules/README.md
diff --git a/src/test/modules/dummy_index_am/README b/src/test/modules/dummy_index_am/README.md
similarity index 100%
rename from src/test/modules/dummy_index_am/README
rename to src/test/modules/dummy_index_am/README.md
diff --git a/src/test/modules/dummy_seclabel/README b/src/test/modules/dummy_seclabel/README.md
similarity index 100%
rename from src/test/modules/dummy_seclabel/README
rename to src/test/modules/dummy_seclabel/README.md
diff --git a/src/test/modules/libpq_pipeline/README b/src/test/modules/libpq_pipeline/README.md
similarity index 100%
rename from src/test/modules/libpq_pipeline/README
rename to src/test/modules/libpq_pipeline/README.md
diff --git a/src/test/modules/plsample/README b/src/test/modules/plsample/README.md
similarity index 100%
rename from src/test/modules/plsample/README
rename to src/test/modules/plsample/README.md
diff --git a/src/test/modules/spgist_name_ops/README b/src/test/modules/spgist_name_ops/README.md
similarity index 100%
rename from src/test/modules/spgist_name_ops/README
rename to src/test/modules/spgist_name_ops/README.md
diff --git a/src/test/modules/test_bloomfilter/README b/src/test/modules/test_bloomfilter/README.md
similarity index 100%
rename from src/test/modules/test_bloomfilter/README
rename to src/test/modules/test_bloomfilter/README.md
diff --git a/src/test/modules/test_ddl_deparse/README b/src/test/modules/test_ddl_deparse/README.md
similarity index 100%
rename from src/test/modules/test_ddl_deparse/README
rename to src/test/modules/test_ddl_deparse/README.md
diff --git a/src/test/modules/test_ginpostinglist/README b/src/test/modules/test_ginpostinglist/README.md
similarity index 100%
rename from src/test/modules/test_ginpostinglist/README
rename to src/test/modules/test_ginpostinglist/README.md
diff --git a/src/test/modules/test_integerset/README b/src/test/modules/test_integerset/README.md
similarity index 100%
rename from src/test/modules/test_integerset/README
rename to src/test/modules/test_integerset/README.md
diff --git a/src/test/modules/test_json_parser/README b/src/test/modules/test_json_parser/README.md
similarity index 100%
rename from src/test/modules/test_json_parser/README
rename to src/test/modules/test_json_parser/README.md
diff --git a/src/test/modules/test_misc/README b/src/test/modules/test_misc/README.md
similarity index 100%
rename from src/test/modules/test_misc/README
rename to src/test/modules/test_misc/README.md
diff --git a/src/test/modules/test_oat_hooks/README b/src/test/modules/test_oat_hooks/README.md
similarity index 100%
rename from src/test/modules/test_oat_hooks/README
rename to src/test/modules/test_oat_hooks/README.md
diff --git a/src/test/modules/test_parser/README b/src/test/modules/test_parser/README.md
similarity index 100%
rename from src/test/modules/test_parser/README
rename to src/test/modules/test_parser/README.md
diff --git a/src/test/modules/test_pg_dump/README b/src/test/modules/test_pg_dump/README.md
similarity index 100%
rename from src/test/modules/test_pg_dump/README
rename to src/test/modules/test_pg_dump/README.md
diff --git a/src/test/modules/test_predtest/README b/src/test/modules/test_predtest/README.md
similarity index 100%
rename from src/test/modules/test_predtest/README
rename to src/test/modules/test_predtest/README.md
diff --git a/src/test/modules/test_rbtree/README b/src/test/modules/test_rbtree/README.md
similarity index 100%
rename from src/test/modules/test_rbtree/README
rename to src/test/modules/test_rbtree/README.md
diff --git a/src/test/modules/test_regex/README b/src/test/modules/test_regex/README.md
similarity index 100%
rename from src/test/modules/test_regex/README
rename to src/test/modules/test_regex/README.md
diff --git a/src/test/modules/test_rls_hooks/README b/src/test/modules/test_rls_hooks/README.md
similarity index 100%
rename from src/test/modules/test_rls_hooks/README
rename to src/test/modules/test_rls_hooks/README.md
diff --git a/src/test/modules/test_shm_mq/README b/src/test/modules/test_shm_mq/README.md
similarity index 100%
rename from src/test/modules/test_shm_mq/README
rename to src/test/modules/test_shm_mq/README.md
diff --git a/src/test/modules/unsafe_tests/README b/src/test/modules/unsafe_tests/README.md
similarity index 100%
rename from src/test/modules/unsafe_tests/README
rename to src/test/modules/unsafe_tests/README.md
diff --git a/src/test/modules/xid_wraparound/README b/src/test/modules/xid_wraparound/README.md
similarity index 100%
rename from src/test/modules/xid_wraparound/README
rename to src/test/modules/xid_wraparound/README.md
diff --git a/src/test/perl/README b/src/test/perl/README.md
similarity index 100%
rename from src/test/perl/README
rename to src/test/perl/README.md
diff --git a/src/test/recovery/README b/src/test/recovery/README.md
similarity index 100%
rename from src/test/recovery/README
rename to src/test/recovery/README.md
diff --git a/src/test/regress/README b/src/test/regress/README.md
similarity index 100%
rename from src/test/regress/README
rename to src/test/regress/README.md
diff --git a/src/test/ssl/README b/src/test/ssl/README.md
similarity index 100%
rename from src/test/ssl/README
rename to src/test/ssl/README.md
diff --git a/src/test/subscription/README b/src/test/subscription/README.md
similarity index 100%
rename from src/test/subscription/README
rename to src/test/subscription/README.md
diff --git a/src/timezone/README b/src/timezone/README.md
similarity index 100%
rename from src/timezone/README
rename to src/timezone/README.md
diff --git a/src/timezone/tznames/README b/src/timezone/tznames/README.md
similarity index 100%
rename from src/timezone/tznames/README
rename to src/timezone/tznames/README.md
diff --git a/src/tools/ci/README b/src/tools/ci/README.md
similarity index 100%
rename from src/tools/ci/README
rename to src/tools/ci/README.md
diff --git a/src/tools/ifaddrs/README b/src/tools/ifaddrs/README.md
similarity index 100%
rename from src/tools/ifaddrs/README
rename to src/tools/ifaddrs/README.md
diff --git a/src/tools/pg_bsd_indent/README b/src/tools/pg_bsd_indent/README.md
similarity index 100%
rename from src/tools/pg_bsd_indent/README
rename to src/tools/pg_bsd_indent/README.md
diff --git a/src/tools/pginclude/README b/src/tools/pginclude/README.md
similarity index 100%
rename from src/tools/pginclude/README
rename to src/tools/pginclude/README.md
diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README.md
similarity index 100%
rename from src/tools/pgindent/README
rename to src/tools/pgindent/README.md
diff --git a/src/tutorial/README b/src/tutorial/README.md
similarity index 100%
rename from src/tutorial/README
rename to src/tutorial/README.md
--
2.39.3 (Apple Git-146)
[application/octet-stream] v2-0002-Convert-internal-documentation-to-markdown-Conver.patch (128.0K, ../../[email protected]/3-v2-0002-Convert-internal-documentation-to-markdown-Conver.patch)
download | inline diff:
From d11e6606036edeb07cfd6915e173ac48752fecf3 Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <[email protected]>
Date: Mon, 8 Apr 2024 14:54:42 +0200
Subject: [PATCH v2 2/2] Convert internal documentation to markdown -
Conversion
This patchset intends to achieve proper Markdown rendering
with the least amount of changes to the source documents.
This patch mainly changes whitespace in order to keep code
from rendering as text, and to separate lists etc. There
are few notable exceptions:
* A few bulletlist characters are changed
* In some files the section headlines lacked underlining
* Very few instances required backticks in order to keep
text being rendered in italics.
---
contrib/start-scripts/macos/README.md | 9 +-
src/backend/access/gin/README.md | 104 ++++----
src/backend/access/gist/README.md | 148 +++++------
src/backend/access/hash/README.md | 76 +++---
src/backend/access/heap/README.tuplock.md | 10 +-
src/backend/access/spgist/README.md | 80 +++---
src/backend/access/transam/README.md | 56 ++---
src/backend/lib/README.md | 22 +-
src/backend/libpq/README.SSL.md | 86 +++----
src/backend/optimizer/README.md | 244 ++++++++++---------
src/backend/optimizer/plan/README.md | 106 ++++----
src/backend/parser/README.md | 42 ++--
src/backend/regex/README.md | 46 ++--
src/backend/snowball/README.md | 14 +-
src/backend/storage/freespace/README.md | 56 ++---
src/backend/storage/lmgr/README-SSI.md | 134 +++++-----
src/backend/utils/fmgr/README.md | 77 +++---
src/backend/utils/mb/README.md | 22 +-
src/backend/utils/misc/README.md | 62 +++--
src/backend/utils/mmgr/README.md | 10 +-
src/backend/utils/resowner/README.md | 52 ++--
src/interfaces/ecpg/preproc/README.parser.md | 29 ++-
src/port/README.md | 2 +-
src/test/isolation/README.md | 30 ++-
src/test/kerberos/README.md | 4 +
src/test/locale/README.md | 3 +
src/test/modules/dummy_seclabel/README.md | 12 +-
src/test/modules/test_parser/README.md | 74 +++---
src/test/modules/test_regex/README.md | 2 +-
src/test/modules/test_rls_hooks/README.md | 8 +-
src/test/modules/test_shm_mq/README.md | 14 +-
src/test/recovery/README.md | 8 +-
src/test/ssl/README.md | 30 ++-
src/timezone/README.md | 6 +-
src/timezone/tznames/README.md | 8 +-
src/tools/ci/README.md | 4 +-
src/tools/pg_bsd_indent/README.md | 4 +-
src/tools/pgindent/README.md | 9 +-
38 files changed, 897 insertions(+), 806 deletions(-)
diff --git a/contrib/start-scripts/macos/README.md b/contrib/start-scripts/macos/README.md
index c4f2d9a270..8fe6efb657 100644
--- a/contrib/start-scripts/macos/README.md
+++ b/contrib/start-scripts/macos/README.md
@@ -15,10 +15,13 @@ if you plan to run the Postgres server under some user name other
than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
-/Library/LaunchDaemons/. You must do this as root:
- sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
-because the file will be ignored if it is not root-owned.
+ /Library/LaunchDaemons/. You must do this as root:
+
+ sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
+
+ because the file will be ignored if it is not root-owned.
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
+
sudo launchctl load /Library/LaunchDaemons/org.postgresql.postgres.plist
diff --git a/src/backend/access/gin/README.md b/src/backend/access/gin/README.md
index b080731621..c25d45ad31 100644
--- a/src/backend/access/gin/README.md
+++ b/src/backend/access/gin/README.md
@@ -40,7 +40,7 @@ Core PostgreSQL includes built-in Gin support for one-dimensional arrays
Synopsis
--------
-=# create index txt_idx on aa using gin(a);
+ =# create index txt_idx on aa using gin(a);
Features
--------
@@ -120,19 +120,21 @@ be, in which case a null bitmap is present as usual. (As usual for index
tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
* If the key datum is null (ie, IndexTupleHasNulls() is true), then
-just after the nominal index data (ie, at offset IndexInfoFindDataOffset
-or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
-the "category" of the null entry. These are the possible categories:
+ just after the nominal index data (ie, at offset IndexInfoFindDataOffset
+ or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
+ the "category" of the null entry. These are the possible categories:
+
1 = ordinary null key value extracted from an indexable item
2 = placeholder for zero-key indexable item
3 = placeholder for null indexable item
-Placeholder null entries are inserted into the index because otherwise
-there would be no index entry at all for an empty or null indexable item,
-which would mean that full index scans couldn't be done and various corner
-cases would give wrong answers. The different categories of null entries
-are treated as distinct keys by the btree, but heap itempointers for the
-same category of null entry are merged into one index entry just as happens
-with ordinary key entries.
+
+ Placeholder null entries are inserted into the index because otherwise
+ there would be no index entry at all for an empty or null indexable item,
+ which would mean that full index scans couldn't be done and various corner
+ cases would give wrong answers. The different categories of null entries
+ are treated as distinct keys by the btree, but heap itempointers for the
+ same category of null entry are merged into one index entry just as happens
+ with ordinary key entries.
* In a key entry at the btree leaf level, at the next SHORTALIGN boundary,
there is a list of item pointers, in compressed format (see Posting List
@@ -325,11 +327,11 @@ getting them on the next page.
The picture below shows tree state after finding the leaf page. Lower case
letters depicts tree pages. 'S' depicts shared lock on the page.
- a
- / | \
- b c d
- / | \ | \ | \
- eS f g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS f g h i j k
### Steping right
@@ -346,11 +348,11 @@ concurrently and doesn't delete right sibling accordingly.
The picture below shows two pages locked at once during stepping right.
- a
- / | \
- b c d
- / | \ | \ | \
- eS fS g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS fS g h i j k
### Insert
@@ -365,11 +367,11 @@ The picture below shows leaf page locked in exclusive mode and ready for
insertion. 'P' and 'E' depict pin and exclusive lock correspondingly.
- aP
- / | \
- b cP d
- / | \ | \ | \
- e f g hE i j k
+ aP
+ / | \
+ b cP d
+ / | \ | \ | \
+ e f g hE i j k
If insert causes a page split, the parent is locked in exclusive mode before
@@ -379,11 +381,11 @@ parent and child pages at once starting from child.
The picture below shows tree state after leaf page split. 'q' is new page
produced by split. Parent 'c' is about to have downlink inserted.
- aP
- / | \
- b cE d
- / | \ / | \ | \
- e f g hE q i j k
+ aP
+ / | \
+ b cE d
+ / | \ / | \ | \
+ e f g hE q i j k
### Page deletion
@@ -404,11 +406,11 @@ we locked it.
The picture below shows tree state after page deletion algorithm traversed to
leftmost leaf of the tree.
- aE
- / | \
- bE c d
- / | \ | \ | \
- eE f g h i j k
+ aE
+ / | \
+ bE c d
+ / | \ | \ | \
+ eE f g h i j k
Deletion algorithm keeps exclusive locks on left siblings of pages comprising
currently investigated path. Thus, if current page is to be removed, all
@@ -436,21 +438,21 @@ The picture below shows tree state after page deletion algorithm further
traversed the tree. Currently investigated path is 'a-c-h'. Left siblings 'b'
and 'g' of 'c' and 'h' correspondingly are also exclusively locked.
- aE
- / | \
- bE cE d
- / | \ | \ | \
- e f gE hE i j k
+ aE
+ / | \
+ bE cE d
+ / | \ | \ | \
+ e f gE hE i j k
The next picture shows tree state after page 'h' was deleted. It's marked with
'deleted' flag and newest xid, which might visit it. Downlink from 'c' to 'h'
is also deleted.
- aE
- / | \
- bE cE d
- / | \ \ | \
- e f gE hD iE j k
+ aE
+ / | \
+ bE cE d
+ / | \ \ | \
+ e f gE hD iE j k
However, it's still possible that concurrent reader has seen downlink from 'c'
to 'h' before we deleted it. In that case this reader will step right from 'h'
@@ -463,11 +465,11 @@ The next picture shows tree state after 'i' and 'c' was deleted. Internal page
investigation is 'a-d-j'. Pages 'b' and 'g' are locked as self siblings of 'd'
and 'j'.
- aE
- / \
- bE cD dE
- / | \ | \
- e f gE hD iD jE k
+ aE
+ / \
+ bE cD dE
+ / | \ | \
+ e f gE hD iD jE k
During the replay of page deletion at standby, the page's left sibling, the
target page, and its parent, are locked in that order. This order guarantees
diff --git a/src/backend/access/gist/README.md b/src/backend/access/gist/README.md
index 8015ff19f0..af082fc2bb 100644
--- a/src/backend/access/gist/README.md
+++ b/src/backend/access/gist/README.md
@@ -183,70 +183,70 @@ operation.
findPath is a subroutine of findParent, used when the correct parent page
can't be found by following the rightlinks at the parent level:
-findPath( stack item )
- push stack, [root, 0, 0] // page, LSN, parent
- while( stack )
- ptr = top of stack
- latch( ptr->page, S-mode )
- if ( ptr->parent->page->lsn < ptr->page->nsn )
- push stack, [ ptr->page->rightlink, 0, ptr->parent ]
- end
- for( each tuple on page )
- if ( tuple->pagepointer == item->page )
- return stack
- else
- add to stack at the end [tuple->pagepointer,0, ptr]
+ findPath( stack item )
+ push stack, [root, 0, 0] // page, LSN, parent
+ while( stack )
+ ptr = top of stack
+ latch( ptr->page, S-mode )
+ if ( ptr->parent->page->lsn < ptr->page->nsn )
+ push stack, [ ptr->page->rightlink, 0, ptr->parent ]
+ end
+ for( each tuple on page )
+ if ( tuple->pagepointer == item->page )
+ return stack
+ else
+ add to stack at the end [tuple->pagepointer,0, ptr]
+ end
end
+ unlatch( ptr->page )
+ pop stack
end
- unlatch( ptr->page )
- pop stack
- end
gistFindCorrectParent is used to re-find the parent of a page during
insertion. It might have migrated to the right since we traversed down the
tree because of page splits.
-findParent( stack item )
- parent = item->parent
- if ( parent->page->lsn != parent->lsn )
- while(true)
- search parent tuple on parent->page, if found the return
- rightlink = parent->page->rightlink
- unlatch( parent->page )
- if ( rightlink is incorrect )
- break loop
+ findParent( stack item )
+ parent = item->parent
+ if ( parent->page->lsn != parent->lsn )
+ while(true)
+ search parent tuple on parent->page, if found the return
+ rightlink = parent->page->rightlink
+ unlatch( parent->page )
+ if ( rightlink is incorrect )
+ break loop
+ end
+ parent->page = rightlink
+ latch( parent->page, X-mode )
end
- parent->page = rightlink
+ newstack = findPath( item->parent )
+ replace part of stack to new one
latch( parent->page, X-mode )
+ return findParent( item )
end
- newstack = findPath( item->parent )
- replace part of stack to new one
- latch( parent->page, X-mode )
- return findParent( item )
- end
pageSplit function decides how to distribute keys to the new pages after
page split:
-pageSplit(page, allkeys)
- (lkeys, rkeys) = pickSplit( allkeys )
- if ( page is root )
- lpage = new page
- else
- lpage = page
- rpage = new page
- if ( no space left on rpage )
- newkeys = pageSplit( rpage, rkeys )
- else
- push newkeys, union(rkeys)
- end
- if ( no space left on lpage )
- push newkeys, pageSplit( lpage, lkeys )
- else
- push newkeys, union(lkeys)
- end
- return newkeys
+ pageSplit(page, allkeys)
+ (lkeys, rkeys) = pickSplit( allkeys )
+ if ( page is root )
+ lpage = new page
+ else
+ lpage = page
+ rpage = new page
+ if ( no space left on rpage )
+ newkeys = pageSplit( rpage, rkeys )
+ else
+ push newkeys, union(rkeys)
+ end
+ if ( no space left on lpage )
+ push newkeys, pageSplit( lpage, lkeys )
+ else
+ push newkeys, union(lkeys)
+ end
+ return newkeys
@@ -302,18 +302,18 @@ In the algorithm, levels are numbered so that leaf pages have level zero,
and internal node levels count up from 1. This numbering ensures that a page's
level number never changes, even when the root page is split.
-Level Tree
+ Level Tree
-3 *
- / \
-2 * *
- / | \ / | \
-1 * * * * * *
- / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o
+ 3 *
+ / \
+ 2 * *
+ / | \ / | \
+ 1 * * * * * *
+ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o
-* - internal page
-o - leaf page
+ * - internal page
+ o - leaf page
Internal pages that belong to certain levels have buffers associated with
them. Leaf pages never have buffers. Which levels have buffers is controlled
@@ -322,17 +322,17 @@ have buffers, while others do not. For example, if level_step = 2, then
pages on levels 2, 4, 6, ... have buffers. If level_step = 1 then every
internal page has a buffer.
-Level Tree (level_step = 1) Tree (level_step = 2)
+ Level Tree (level_step = 1) Tree (level_step = 2)
-3 * *
- / \ / \
-2 *(b) *(b) *(b) *(b)
- / | \ / | \ / | \ / | \
-1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
- / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o o o o o o o o o o o o o
+ 3 * *
+ / \ / \
+ 2 *(b) *(b) *(b) *(b)
+ / | \ / | \ / | \ / | \
+ 1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
+ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o o o o o o o o o o o o o
-(b) - buffer
+ (b) - buffer
Logically, a buffer is just bunch of tuples. Physically, it is divided in
pages, backed by a temporary file. Each buffer can be in one of two states:
@@ -365,7 +365,7 @@ pages where the tuples finally land on get cached too. If there are, the last
buffer page of each buffer below is kept in memory. This is illustrated in
the figures below:
- Buffer being emptied to
+ Buffer being emptied to
lower-level buffers Buffer being emptied to leaf pages
+(fb) +(fb)
@@ -374,11 +374,11 @@ the figures below:
/ \ / \ / \ / \
*(ab) *(ab) *(ab) *(ab) x x x x
-+ - cached internal page
-x - cached leaf page
-* - non-cached internal page
-(fb) - buffer being emptied
-(ab) - buffers being appended to, with last page in memory
+ + - cached internal page
+ x - cached leaf page
+ * - non-cached internal page
+ (fb) - buffer being emptied
+ (ab) - buffers being appended to, with last page in memory
In the beginning of the index build, the level-step is chosen so that all those
pages involved in emptying one buffer fit in cache, so after each of those
diff --git a/src/backend/access/hash/README.md b/src/backend/access/hash/README.md
index 13dc59c124..a5df13a68a 100644
--- a/src/backend/access/hash/README.md
+++ b/src/backend/access/hash/README.md
@@ -248,24 +248,24 @@ track of available overflow pages.
The reader algorithm is:
- lock the primary bucket page of the target bucket
- if the target bucket is still being populated by a split:
- release the buffer content lock on current bucket page
- pin and acquire the buffer content lock on old bucket in shared mode
- release the buffer content lock on old bucket, but not pin
- retake the buffer content lock on new bucket
- arrange to scan the old bucket normally and the new bucket for
- tuples which are not moved-by-split
--- then, per read request:
- reacquire content lock on current page
- step to next page if necessary (no chaining of content locks, but keep
- the pin on the primary bucket throughout the scan)
- save all the matching tuples from current index page into an items array
- release pin and content lock (but if it is primary bucket page retain
- its pin till the end of the scan)
- get tuple from an item array
--- at scan shutdown:
- release all pins still held
+ lock the primary bucket page of the target bucket
+ if the target bucket is still being populated by a split:
+ release the buffer content lock on current bucket page
+ pin and acquire the buffer content lock on old bucket in shared mode
+ release the buffer content lock on old bucket, but not pin
+ retake the buffer content lock on new bucket
+ arrange to scan the old bucket normally and the new bucket for
+ tuples which are not moved-by-split
+ -- then, per read request:
+ reacquire content lock on current page
+ step to next page if necessary (no chaining of content locks, but keep
+ the pin on the primary bucket throughout the scan)
+ save all the matching tuples from current index page into an items array
+ release pin and content lock (but if it is primary bucket page retain
+ its pin till the end of the scan)
+ get tuple from an item array
+ -- at scan shutdown:
+ release all pins still held
Holding the buffer pin on the primary bucket page for the whole scan prevents
the reader's current-tuple pointer from being invalidated by splits or
@@ -288,8 +288,8 @@ which this bucket is formed by split.
The insertion algorithm is rather similar:
lock the primary bucket page of the target bucket
--- (so far same as reader, except for acquisition of buffer content lock in
- exclusive mode on primary bucket page)
+ -- (so far same as reader, except for acquisition of buffer content lock in
+ exclusive mode on primary bucket page)
if the bucket-being-split flag is set for a bucket and pin count on it is
one, then finish the split
release the buffer content lock on current bucket
@@ -465,24 +465,24 @@ overflow page to the free pool.
Obtaining an overflow page:
- take metapage content lock in exclusive mode
- determine next bitmap page number; if none, exit loop
- release meta page content lock
- pin bitmap page and take content lock in exclusive mode
- search for a free page (zero bit in bitmap)
- if found:
- set bit in bitmap
- mark bitmap page dirty
- take metapage buffer content lock in exclusive mode
- if first-free-bit value did not change,
- update it and mark meta page dirty
- else (not found):
- release bitmap page buffer content lock
- loop back to try next bitmap page, if any
--- here when we have checked all bitmap pages; we hold meta excl. lock
- extend index to add another overflow page; update meta information
- mark meta page dirty
- return page number
+ take metapage content lock in exclusive mode
+ determine next bitmap page number; if none, exit loop
+ release meta page content lock
+ pin bitmap page and take content lock in exclusive mode
+ search for a free page (zero bit in bitmap)
+ if found:
+ set bit in bitmap
+ mark bitmap page dirty
+ take metapage buffer content lock in exclusive mode
+ if first-free-bit value did not change,
+ update it and mark meta page dirty
+ else (not found):
+ release bitmap page buffer content lock
+ loop back to try next bitmap page, if any
+ -- here when we have checked all bitmap pages; we hold meta excl. lock
+ extend index to add another overflow page; update meta information
+ mark meta page dirty
+ return page number
It is slightly annoying to release and reacquire the metapage lock
multiple times, but it seems best to do it that way to minimize loss of
diff --git a/src/backend/access/heap/README.tuplock.md b/src/backend/access/heap/README.tuplock.md
index 6441e8baf0..5a91b4fddd 100644
--- a/src/backend/access/heap/README.tuplock.md
+++ b/src/backend/access/heap/README.tuplock.md
@@ -62,11 +62,11 @@ the tuple without changing its key.
The conflict table is:
- UPDATE NO KEY UPDATE SHARE KEY SHARE
-UPDATE conflict conflict conflict conflict
-NO KEY UPDATE conflict conflict conflict
-SHARE conflict conflict
-KEY SHARE conflict
+ UPDATE NO KEY UPDATE SHARE KEY SHARE
+ UPDATE conflict conflict conflict conflict
+ NO KEY UPDATE conflict conflict conflict
+ SHARE conflict conflict
+ KEY SHARE conflict
When there is a single locker in a tuple, we can just store the locking info
in the tuple itself. We do this by storing the locker's Xid in XMAX, and
diff --git a/src/backend/access/spgist/README.md b/src/backend/access/spgist/README.md
index 7117e02c77..2b890e4f8e 100644
--- a/src/backend/access/spgist/README.md
+++ b/src/backend/access/spgist/README.md
@@ -13,7 +13,7 @@ few disk pages, even if it traverses many nodes.
COMMON STRUCTURE DESCRIPTION
-
+----------------------------
Logically, an SP-GiST tree is a set of tuples, each of which can be either
an inner or leaf tuple. Each inner tuple contains "nodes", which are
(label,pointer) pairs, where the pointer (ItemPointerData) is a pointer to
@@ -58,27 +58,27 @@ pages.
An inner tuple consists of:
- optional prefix value - all successors must be consistent with it.
- Example:
- radix tree - prefix value is a common prefix string
- quad tree - centroid
- k-d tree - one coordinate
+ optional prefix value - all successors must be consistent with it.
+ Example:
+ radix tree - prefix value is a common prefix string
+ quad tree - centroid
+ k-d tree - one coordinate
- list of nodes, where node is a (label, pointer) pair.
- Example of a label: a single character for radix tree
+ list of nodes, where node is a (label, pointer) pair.
+ Example of a label: a single character for radix tree
A leaf tuple consists of:
- a leaf value
- Example:
- radix tree - the rest of string (postfix)
- quad and k-d tree - the point itself
+ a leaf value
+ Example:
+ radix tree - the rest of string (postfix)
+ quad and k-d tree - the point itself
- ItemPointer to the corresponding heap tuple
- nextOffset number of next leaf tuple in a chain on a leaf page
+ ItemPointer to the corresponding heap tuple
+ nextOffset number of next leaf tuple in a chain on a leaf page
- optional nulls bitmask
- optional INCLUDE-column values
+ optional nulls bitmask
+ optional INCLUDE-column values
For compatibility with pre-v14 indexes, a leaf tuple has a nulls bitmask
only if there are null values (among the leaf value and the INCLUDE values)
@@ -90,7 +90,7 @@ code can be used.
NULLS HANDLING
-
+--------------
We assume that SPGiST-indexable operators are strict (can never succeed for
null inputs). It is still desirable to index nulls, so that whole-table
indexscans are possible and so that "x IS NULL" can be implemented by an
@@ -104,27 +104,27 @@ AllTheSame cases in the normal tree.
INSERTION ALGORITHM
-
+-------------------
Insertion algorithm is designed to keep the tree in a consistent state at
any moment. Here is a simplified insertion algorithm specification
(numbers refer to notes below):
- Start with the first tuple on the root page (1)
-
- loop:
- if (page is leaf) then
- if (enough space)
- insert on page and exit (5)
- else (7)
- call PickSplitFn() (2)
- end if
- else
- switch (chooseFn())
- case MatchNode - descend through selected node
- case AddNode - add node and then retry chooseFn (3, 6)
- case SplitTuple - split inner tuple to prefix and postfix, then
- retry chooseFn with the prefix tuple (4, 6)
- end if
+ Start with the first tuple on the root page (1)
+
+ loop:
+ if (page is leaf) then
+ if (enough space)
+ insert on page and exit (5)
+ else (7)
+ call PickSplitFn() (2)
+ end if
+ else
+ switch (chooseFn())
+ case MatchNode - descend through selected node
+ case AddNode - add node and then retry chooseFn (3, 6)
+ case SplitTuple - split inner tuple to prefix and postfix, then
+ retry chooseFn with the prefix tuple (4, 6)
+ end if
Notes:
@@ -160,7 +160,7 @@ the following notation, where tuple's id is just for discussion (no such id
is actually stored):
inner tuple: {tuple id}(prefix string)[ comma separated list of node labels ]
-leaf tuple: {tuple id}<value>
+leaf tuple: {tuple id}< value >
Suppose we need to insert string 'www.gogo.com' into inner tuple
@@ -215,7 +215,7 @@ space utilization, but doesn't change the basis of the algorithm.
CONCURRENCY
-
+-----------
While descending the tree, the insertion algorithm holds exclusive lock on
two tree levels at a time, ie both parent and child pages (but parent and
child pages can be the same, see notes above). There is a possibility of
@@ -267,7 +267,7 @@ been flushed out of the system.
DEAD TUPLES
-
+-----------
Tuples on leaf pages can be in one of four states:
SPGIST_LIVE: normal, live pointer to a heap tuple.
@@ -319,7 +319,7 @@ remove unused inner tuples.
VACUUM
-
+------
VACUUM (or more precisely, spgbulkdelete) performs a single sequential scan
over the entire index. On both leaf and inner pages, we can convert old
REDIRECT tuples into PLACEHOLDER status, and then remove any PLACEHOLDERs
@@ -374,7 +374,7 @@ space map, and gather statistics.
LAST USED PAGE MANAGEMENT
-
+-------------------------
The list of last used pages contains four pages - a leaf page and three
inner pages, one from each "triple parity" group. (Actually, there's one
such list for the main tree and a separate one for the nulls tree.) This
@@ -384,6 +384,6 @@ critical, because we could allocate a new page at any moment.
AUTHORS
-
+-------
Teodor Sigaev <[email protected]>
Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/transam/README.md b/src/backend/access/transam/README.md
index 28d196cf62..9a2181aa30 100644
--- a/src/backend/access/transam/README.md
+++ b/src/backend/access/transam/README.md
@@ -59,27 +59,27 @@ For example, consider the following sequence of user commands:
In the main processing loop, this results in the following function call
sequence:
- / StartTransactionCommand;
- / StartTransaction;
-1) < ProcessUtility; << BEGIN
- \ BeginTransactionBlock;
- \ CommitTransactionCommand;
-
- / StartTransactionCommand;
-2) / PortalRunSelect; << SELECT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
-3) / ProcessQuery; << INSERT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
- / ProcessUtility; << COMMIT
-4) < EndTransactionBlock;
- \ CommitTransactionCommand;
- \ CommitTransaction;
+ / StartTransactionCommand;
+ / StartTransaction;
+ 1) < ProcessUtility; << BEGIN
+ \ BeginTransactionBlock;
+ \ CommitTransactionCommand;
+
+ / StartTransactionCommand;
+ 2) / PortalRunSelect; << SELECT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ 3) / ProcessQuery; << INSERT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ / ProcessUtility; << COMMIT
+ 4) < EndTransactionBlock;
+ \ CommitTransactionCommand;
+ \ CommitTransaction;
The point of this example is to demonstrate the need for
StartTransactionCommand and CommitTransactionCommand to be state smart -- they
@@ -100,12 +100,12 @@ Transaction aborts can occur in two ways:
The reason we have to distinguish them is illustrated by the following two
situations:
- case 1 case 2
- ------ ------
-1) user types BEGIN 1) user types BEGIN
-2) user does something 2) user does something
-3) user does not like what 3) system aborts for some reason
- she sees and types ABORT (syntax error, etc)
+ case 1 case 2
+ ------ ------
+ 1) user types BEGIN 1) user types BEGIN
+ 2) user does something 2) user does something
+ 3) user does not like what 3) system aborts for some reason
+ she sees and types ABORT (syntax error, etc)
In case 1, we want to abort the transaction and return to the default state.
In case 2, there may be more commands coming our way which are part of the
@@ -171,7 +171,7 @@ CommitTransactionCommand, the real work is done. The main point of doing
things this way is that if we get an error while popping state stack entries,
the remaining stack entries still show what we need to do to finish up.
-In the case of ROLLBACK TO <savepoint>, we abort all the subtransactions up
+In the case of ROLLBACK TO < savepoint >, we abort all the subtransactions up
through the one identified by the savepoint name, and then re-create that
subtransaction level with the same name. So it's a completely new
subtransaction as far as the internals are concerned.
diff --git a/src/backend/lib/README.md b/src/backend/lib/README.md
index f2fb591237..fc8e1aa1f7 100644
--- a/src/backend/lib/README.md
+++ b/src/backend/lib/README.md
@@ -1,27 +1,27 @@
This directory contains a general purpose data structures, for use anywhere
in the backend:
-binaryheap.c - a binary heap
+ binaryheap.c - a binary heap
-bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
+ bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
-bloomfilter.c - probabilistic, space-efficient set membership testing
+ bloomfilter.c - probabilistic, space-efficient set membership testing
-dshash.c - concurrent hash tables backed by dynamic shared memory areas
+ dshash.c - concurrent hash tables backed by dynamic shared memory areas
-hyperloglog.c - a streaming cardinality estimator
+ hyperloglog.c - a streaming cardinality estimator
-ilist.c - single and double-linked lists
+ ilist.c - single and double-linked lists
-integerset.c - a data structure for holding large set of integers
+ integerset.c - a data structure for holding large set of integers
-knapsack.c - knapsack problem solver
+ knapsack.c - knapsack problem solver
-pairingheap.c - a pairing heap
+ pairingheap.c - a pairing heap
-rbtree.c - a red-black tree
+ rbtree.c - a red-black tree
-stringinfo.c - an extensible string type
+ stringinfo.c - an extensible string type
Aside from the inherent characteristics of the data structures, there are a
diff --git a/src/backend/libpq/README.SSL.md b/src/backend/libpq/README.SSL.md
index d84a434a6e..98e20498bd 100644
--- a/src/backend/libpq/README.SSL.md
+++ b/src/backend/libpq/README.SSL.md
@@ -3,59 +3,59 @@ src/backend/libpq/README.SSL
SSL
===
->From the servers perspective:
+From the servers perspective:
- Receives StartupPacket
- |
- |
- (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
- | No
- |
- | Yes
+ Receives StartupPacket
+ |
+ |
+ (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
+ | No
+ |
+ | Yes
+ |
+ |
+ (Server compiled with USE_SSL?) ------- Send 'N'
+ | No |
+ | |
+ | Yes Normal startup
+ |
+ |
+ Send 'S'
+ |
+ |
+ Establish SSL
+ |
+ |
+ Normal startup
+
+
+
+
+
+From the clients perspective (v6.6 client _with_ SSL):
+
+
+ Connect
|
|
- (Server compiled with USE_SSL?) ------- Send 'N'
- | No |
- | |
- | Yes Normal startup
+ Send packet with SSL_NEGOTIATE_CODE
|
|
- Send 'S'
+ Receive single char ------- 'S' -------- Establish SSL
+ | |
+ | '<else>' |
+ | Normal startup
|
|
- Establish SSL
+ Is it 'E' for error ------------------- Retry connection
+ | Yes without SSL
+ | No
|
+ Is it 'N' for normal ------------------- Normal startup
+ | Yes
|
- Normal startup
-
-
-
-
-
->From the clients perspective (v6.6 client _with_ SSL):
-
-
- Connect
- |
- |
- Send packet with SSL_NEGOTIATE_CODE
- |
- |
- Receive single char ------- 'S' -------- Establish SSL
- | |
- | '<else>' |
- | Normal startup
- |
- |
- Is it 'E' for error ------------------- Retry connection
- | Yes without SSL
- | No
- |
- Is it 'N' for normal ------------------- Normal startup
- | Yes
- |
- Fail with unknown
+ Fail with unknown
---------------------------------------------------------------------------
diff --git a/src/backend/optimizer/README.md b/src/backend/optimizer/README.md
index 2ab4f3dbf3..5805b61a23 100644
--- a/src/backend/optimizer/README.md
+++ b/src/backend/optimizer/README.md
@@ -254,7 +254,9 @@ the boundary, unless the proposed join is a LEFT join that can associate
into the SpecialJoinInfo's RHS using identity 3.
The use of minimum Relid sets has some pitfalls; consider a query like
+
A leftjoin (B leftjoin (C innerjoin D) on (Pbcd)) on Pa
+
where Pa doesn't mention B/C/D at all. In this case a naive computation
would give the upper leftjoin's min LHS as {A} and min RHS as {C,D} (since
we know that the innerjoin can't associate out of the leftjoin's RHS, and
@@ -262,7 +264,9 @@ enforce that by including its relids in the leftjoin's min RHS). And the
lower leftjoin has min LHS of {B} and min RHS of {C,D}. Given such
information, join_is_legal would think it's okay to associate the upper
join into the lower join's RHS, transforming the query to
+
B leftjoin (A leftjoin (C innerjoin D) on Pa) on (Pbcd)
+
which yields totally wrong answers. We prevent that by forcing the min RHS
for the upper join to include B. This is perhaps overly restrictive, but
such cases don't arise often so it's not clear that it's worth developing a
@@ -359,10 +363,12 @@ side of the full join a Var came from; but that information can be found
elsewhere at need.)
Notionally, a Var having nonempty varnullingrels can be thought of as
+
CASE WHEN any-of-these-outer-joins-produced-a-null-extended-row
THEN NULL
ELSE the-scan-level-value-of-the-column
END
+
It's only notional, because no such calculation is ever done explicitly.
In a finished plan, Vars occurring in scan-level plan nodes represent
the actual table column values, but upper-level Vars are always
@@ -375,14 +381,20 @@ otherwise be essential information for FULL JOIN cases.
Outer join identity 3 (discussed above) complicates this picture
a bit. In the form
+
A leftjoin (B leftjoin C on (Pbc)) on (Pab)
+
all of the Vars in clauses Pbc and Pab will have empty varnullingrels,
but if we start with
+
(A leftjoin B on (Pab)) leftjoin C on (Pbc)
+
then the parser will have marked Pbc's B Vars with the A/B join's
RT index, making this form artificially different from the first.
For discussion's sake, let's denote this marking with a star:
+
(A leftjoin B on (Pab)) leftjoin C on (Pb*c)
+
To cope with this, once we have detected that commuting these joins
is legal, we generate both the Pbc and Pb*c forms of that ON clause,
by either removing or adding the first join's RT index in the B Vars
@@ -553,114 +565,114 @@ Optimizer Functions
The primary entry point is planner().
-planner()
-set up for recursive handling of subqueries
--subquery_planner()
- pull up sublinks and subqueries from rangetable, if possible
- canonicalize qual
- Attempt to simplify WHERE clause to the most useful form; this includes
- flattening nested AND/ORs and detecting clauses that are duplicated in
- different branches of an OR.
- simplify constant expressions
- process sublinks
- convert Vars of outer query levels into Params
---grouping_planner()
- preprocess target list for non-SELECT queries
- handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
- ORDER BY, DISTINCT, LIMIT
----query_planner()
- make list of base relations used in query
- split up the qual into restrictions (a=1) and joins (b=c)
- find qual clauses that enable merge and hash joins
-----make_one_rel()
- set_base_rel_pathlists()
- find seqscan and all index paths for each base relation
- find selectivity of columns used in joins
- make_rel_from_joinlist()
- hand off join subproblems to a plugin, GEQO, or standard_join_search()
-------standard_join_search()
- call join_search_one_level() for each level of join tree needed
- join_search_one_level():
- For each joinrel of the prior level, do make_rels_by_clause_joins()
- if it has join clauses, or make_rels_by_clauseless_joins() if not.
- Also generate "bushy plan" joins between joinrels of lower levels.
- Back at standard_join_search(), generate gather paths if needed for
- each newly constructed joinrel, then apply set_cheapest() to extract
- the cheapest path for it.
- Loop back if this wasn't the top join level.
- Back at grouping_planner:
- do grouping (GROUP BY) and aggregation
- do window functions
- make unique (DISTINCT)
- do sorting (ORDER BY)
- do limit (LIMIT/OFFSET)
-Back at planner():
-convert finished Path tree into a Plan tree
-do final cleanup after planning
+ planner()
+ set up for recursive handling of subqueries
+ -subquery_planner()
+ pull up sublinks and subqueries from rangetable, if possible
+ canonicalize qual
+ Attempt to simplify WHERE clause to the most useful form; this includes
+ flattening nested AND/ORs and detecting clauses that are duplicated in
+ different branches of an OR.
+ simplify constant expressions
+ process sublinks
+ convert Vars of outer query levels into Params
+ --grouping_planner()
+ preprocess target list for non-SELECT queries
+ handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
+ ORDER BY, DISTINCT, LIMIT
+ ---query_planner()
+ make list of base relations used in query
+ split up the qual into restrictions (a=1) and joins (b=c)
+ find qual clauses that enable merge and hash joins
+ ----make_one_rel()
+ set_base_rel_pathlists()
+ find seqscan and all index paths for each base relation
+ find selectivity of columns used in joins
+ make_rel_from_joinlist()
+ hand off join subproblems to a plugin, GEQO, or standard_join_search()
+ ------standard_join_search()
+ call join_search_one_level() for each level of join tree needed
+ join_search_one_level():
+ For each joinrel of the prior level, do make_rels_by_clause_joins()
+ if it has join clauses, or make_rels_by_clauseless_joins() if not.
+ Also generate "bushy plan" joins between joinrels of lower levels.
+ Back at standard_join_search(), generate gather paths if needed for
+ each newly constructed joinrel, then apply set_cheapest() to extract
+ the cheapest path for it.
+ Loop back if this wasn't the top join level.
+ Back at grouping_planner:
+ do grouping (GROUP BY) and aggregation
+ do window functions
+ make unique (DISTINCT)
+ do sorting (ORDER BY)
+ do limit (LIMIT/OFFSET)
+ Back at planner():
+ convert finished Path tree into a Plan tree
+ do final cleanup after planning
Optimizer Data Structures
-------------------------
-PlannerGlobal - global information for a single planner invocation
-
-PlannerInfo - information for planning a particular Query (we make
- a separate PlannerInfo node for each sub-Query)
-
-RelOptInfo - a relation or joined relations
-
- RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
- (note the same structure is used for restriction and
- join clauses)
-
- Path - every way to generate a RelOptInfo(sequential,index,joins)
- A plain Path node can represent several simple plans, per its pathtype:
- T_SeqScan - sequential scan
- T_SampleScan - tablesample scan
- T_FunctionScan - function-in-FROM scan
- T_TableFuncScan - table function scan
- T_ValuesScan - VALUES scan
- T_CteScan - CTE (WITH) scan
- T_NamedTuplestoreScan - ENR scan
- T_WorkTableScan - scan worktable of a recursive CTE
- T_Result - childless Result plan node (used for FROM-less SELECT)
- IndexPath - index scan
- BitmapHeapPath - top of a bitmapped index scan
- TidPath - scan by CTID
- TidRangePath - scan a contiguous range of CTIDs
- SubqueryScanPath - scan a subquery-in-FROM
- ForeignPath - scan a foreign table, foreign join or foreign upper-relation
- CustomPath - for custom scan providers
- AppendPath - append multiple subpaths together
- MergeAppendPath - merge multiple subpaths, preserving their common sort order
- GroupResultPath - childless Result plan node (used for degenerate grouping)
- MaterialPath - a Material plan node
- MemoizePath - a Memoize plan node for caching tuples from sub-paths
- UniquePath - remove duplicate rows (either by hashing or sorting)
- GatherPath - collect the results of parallel workers
- GatherMergePath - collect parallel results, preserving their common sort order
- ProjectionPath - a Result plan node with child (used for projection)
- ProjectSetPath - a ProjectSet plan node applied to some sub-path
- SortPath - a Sort plan node applied to some sub-path
- IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
- GroupPath - a Group plan node applied to some sub-path
- UpperUniquePath - a Unique plan node applied to some sub-path
- AggPath - an Agg plan node applied to some sub-path
- GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
- MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
- WindowAggPath - a WindowAgg plan node applied to some sub-path
- SetOpPath - a SetOp plan node applied to some sub-path
- RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
- LockRowsPath - a LockRows plan node applied to some sub-path
- ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
- LimitPath - a Limit plan node applied to some sub-path
- NestPath - nested-loop joins
- MergePath - merge joins
- HashPath - hash joins
-
- EquivalenceClass - a data structure representing a set of values known equal
-
- PathKey - a data structure representing the sort ordering of a path
+ PlannerGlobal - global information for a single planner invocation
+
+ PlannerInfo - information for planning a particular Query (we make
+ a separate PlannerInfo node for each sub-Query)
+
+ RelOptInfo - a relation or joined relations
+
+ RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
+ (note the same structure is used for restriction and
+ join clauses)
+
+ Path - every way to generate a RelOptInfo(sequential,index,joins)
+ A plain Path node can represent several simple plans, per its pathtype:
+ T_SeqScan - sequential scan
+ T_SampleScan - tablesample scan
+ T_FunctionScan - function-in-FROM scan
+ T_TableFuncScan - table function scan
+ T_ValuesScan - VALUES scan
+ T_CteScan - CTE (WITH) scan
+ T_NamedTuplestoreScan - ENR scan
+ T_WorkTableScan - scan worktable of a recursive CTE
+ T_Result - childless Result plan node (used for FROM-less SELECT)
+ IndexPath - index scan
+ BitmapHeapPath - top of a bitmapped index scan
+ TidPath - scan by CTID
+ TidRangePath - scan a contiguous range of CTIDs
+ SubqueryScanPath - scan a subquery-in-FROM
+ ForeignPath - scan a foreign table, foreign join or foreign upper-relation
+ CustomPath - for custom scan providers
+ AppendPath - append multiple subpaths together
+ MergeAppendPath - merge multiple subpaths, preserving their common sort order
+ GroupResultPath - childless Result plan node (used for degenerate grouping)
+ MaterialPath - a Material plan node
+ MemoizePath - a Memoize plan node for caching tuples from sub-paths
+ UniquePath - remove duplicate rows (either by hashing or sorting)
+ GatherPath - collect the results of parallel workers
+ GatherMergePath - collect parallel results, preserving their common sort order
+ ProjectionPath - a Result plan node with child (used for projection)
+ ProjectSetPath - a ProjectSet plan node applied to some sub-path
+ SortPath - a Sort plan node applied to some sub-path
+ IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
+ GroupPath - a Group plan node applied to some sub-path
+ UpperUniquePath - a Unique plan node applied to some sub-path
+ AggPath - an Agg plan node applied to some sub-path
+ GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
+ MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
+ WindowAggPath - a WindowAgg plan node applied to some sub-path
+ SetOpPath - a SetOp plan node applied to some sub-path
+ RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
+ LockRowsPath - a LockRows plan node applied to some sub-path
+ ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
+ LimitPath - a Limit plan node applied to some sub-path
+ NestPath - nested-loop joins
+ MergePath - merge joins
+ HashPath - hash joins
+
+ EquivalenceClass - a data structure representing a set of values known equal
+
+ PathKey - a data structure representing the sort ordering of a path
The optimizer spends a good deal of its time worrying about the ordering
of the tuples returned by a path. The reason this is useful is that by
@@ -909,10 +921,10 @@ of the tuples generated by a particular Path. A path's pathkeys field is a
list of PathKey nodes, where the n'th item represents the n'th sort key of
the result. Each PathKey contains these fields:
- * a reference to an EquivalenceClass
- * a btree opfamily OID (must match one of those in the EC)
- * a sort direction (ascending or descending)
- * a nulls-first-or-last flag
+* a reference to an EquivalenceClass
+* a btree opfamily OID (must match one of those in the EC)
+* a sort direction (ascending or descending)
+* a nulls-first-or-last flag
The EquivalenceClass represents the value being sorted on. Since the
various members of an EquivalenceClass are known equal according to the
@@ -997,9 +1009,11 @@ lists (sort orderings) do not mention the same EquivalenceClass more than
once. For example, in all these cases the second sort column is redundant,
because it cannot distinguish values that are the same according to the
first sort column:
+
SELECT ... ORDER BY x, x
SELECT ... ORDER BY x, x DESC
SELECT ... WHERE x = y ORDER BY x, y
+
Although a user probably wouldn't write "ORDER BY x,x" directly, such
redundancies are more probable once equivalence classes have been
considered. Also, the system may generate redundant pathkey lists when
@@ -1350,14 +1364,14 @@ RelOptInfos are mostly dummy, but their pathlist lists hold all the Paths
considered useful for each step. Currently, we may create these types of
additional RelOptInfos during upper-level planning:
-UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
-UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
-UPPERREL_GROUP_AGG result of grouping/aggregation, if any
-UPPERREL_WINDOW result of window functions, if any
-UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
-UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
-UPPERREL_ORDERED result of ORDER BY, if any
-UPPERREL_FINAL result of any remaining top-level actions
+ UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
+ UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
+ UPPERREL_GROUP_AGG result of grouping/aggregation, if any
+ UPPERREL_WINDOW result of window functions, if any
+ UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
+ UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
+ UPPERREL_ORDERED result of ORDER BY, if any
+ UPPERREL_FINAL result of any remaining top-level actions
UPPERREL_FINAL is used to represent any final processing steps, currently
LockRows (SELECT FOR UPDATE), LIMIT/OFFSET, and ModifyTable. There is no
diff --git a/src/backend/optimizer/plan/README.md b/src/backend/optimizer/plan/README.md
index 013c0f9ea2..93dd422dc1 100644
--- a/src/backend/optimizer/plan/README.md
+++ b/src/backend/optimizer/plan/README.md
@@ -6,32 +6,32 @@ Subselects
Vadim B. Mikheev
-From [email protected] Fri Feb 13 09:01:19 1998
-Received: from renoir.op.net ([email protected] [209.152.193.4])
- by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
- for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
-Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
-Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
-Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
-Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
-Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
-Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
- by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
- for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
- (envelope-from [email protected])
-Message-ID: <[email protected]>
-Date: Fri, 13 Feb 1998 20:50:50 +0700
-From: "Vadim B. Mikheev" <[email protected]>
-Organization: ITTS (Krasnoyarsk)
-X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
-MIME-Version: 1.0
-To: PostgreSQL Developers List <[email protected]>
-Subject: [HACKERS] Subselects are in CVS...
-Content-Type: text/plain; charset=us-ascii
-Content-Transfer-Encoding: 7bit
-Sender: [email protected]
-Precedence: bulk
-Status: OR
+ From [email protected] Fri Feb 13 09:01:19 1998
+ Received: from renoir.op.net ([email protected] [209.152.193.4])
+ by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
+ for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
+ Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
+ Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
+ Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
+ Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
+ Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
+ Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
+ by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
+ for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
+ (envelope-from [email protected])
+ Message-ID: <[email protected]>
+ Date: Fri, 13 Feb 1998 20:50:50 +0700
+ From: "Vadim B. Mikheev" <[email protected]>
+ Organization: ITTS (Krasnoyarsk)
+ X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
+ MIME-Version: 1.0
+ To: PostgreSQL Developers List <[email protected]>
+ Subject: [HACKERS] Subselects are in CVS...
+ Content-Type: text/plain; charset=us-ascii
+ Content-Transfer-Encoding: 7bit
+ Sender: [email protected]
+ Precedence: bulk
+ Status: OR
This is some implementation notes and opened issues...
@@ -95,17 +95,17 @@ vac=> explain select * from tmp where x >= (select max(x2) from test2
where y2 = y and exists (select * from tempx where tx = x));
NOTICE: QUERY PLAN:
-Seq Scan on tmp (cost=40.03 size=101 width=8)
- SubPlan
- ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
- -> Aggregate (cost=2.05 size=0 width=0)
- InitPlan
- ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
- -> Seq Scan on tempx (cost=4.33 size=1 width=4)
- -> Result (cost=2.05 size=0 width=0)
- ^^^^^^ EXISTS subsubquery was transformed into Param
- and so we have Result node here
- -> Index Scan on test2 (cost=2.05 size=1 width=4)
+ Seq Scan on tmp (cost=40.03 size=101 width=8)
+ SubPlan
+ ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
+ -> Aggregate (cost=2.05 size=0 width=0)
+ InitPlan
+ ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
+ -> Seq Scan on tempx (cost=4.33 size=1 width=4)
+ -> Result (cost=2.05 size=0 width=0)
+ ^^^^^^ EXISTS subsubquery was transformed into Param
+ and so we have Result node here
+ -> Index Scan on test2 (cost=2.05 size=1 width=4)
Opened issues.
@@ -131,28 +131,28 @@ Results of some test. TMP is table with x,y (int4-s), x in 0-9,
y = 100 - x, 1000 tuples (10 duplicates of each tuple). TEST2 is table
with x2, y2 (int4-s), x2 in 1-99, y2 = 100 -x2, 10000 tuples (100 dups).
- Trying
+Trying
-select * from tmp where x >= (select max(x2) from test2 where y2 = y);
+ select * from tmp where x >= (select max(x2) from test2 where y2 = y);
- and
+and
-begin;
-select y as ty, max(x2) as mx into table tsub from test2, tmp
-where y2 = y group by ty;
-vacuum tsub;
-select x, y from tmp, tsub where x >= mx and y = ty;
-drop table tsub;
-end;
+ begin;
+ select y as ty, max(x2) as mx into table tsub from test2, tmp
+ where y2 = y group by ty;
+ vacuum tsub;
+ select x, y from tmp, tsub where x >= mx and y = ty;
+ drop table tsub;
+ end;
- Without index on test2(y2):
+Without index on test2(y2):
-SubSelect -> 320 sec
-Using temp table -> 32 sec
+ SubSelect -> 320 sec
+ Using temp table -> 32 sec
- Having index
+Having index
-SubSelect -> 17 sec (2M of memory)
-Using temp table -> 32 sec (12M of memory: -S 8192)
+ SubSelect -> 17 sec (2M of memory)
+ Using temp table -> 32 sec (12M of memory: -S 8192)
Vadim
diff --git a/src/backend/parser/README.md b/src/backend/parser/README.md
index e0c986a41e..e6016fa430 100644
--- a/src/backend/parser/README.md
+++ b/src/backend/parser/README.md
@@ -7,27 +7,27 @@ This directory does more than tokenize and parse SQL queries. It also
creates Query structures for the various complex queries that are passed
to the optimizer and then executor.
-parser.c things start here
-scan.l break query into tokens
-scansup.c handle escapes in input strings
-gram.y parse the tokens and produce a "raw" parse tree
-analyze.c top level of parse analysis for optimizable queries
-parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
-parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
-parse_coerce.c handle coercing expressions to different data types
-parse_collate.c assign collation information in completed expressions
-parse_cte.c handle Common Table Expressions (WITH clauses)
-parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
-parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
-parse_func.c handle functions, table.column and column identifiers
-parse_merge.c handle MERGE
-parse_node.c create nodes for various structures
-parse_oper.c handle operators in expressions
-parse_param.c handle Params (for the cases used in the core backend)
-parse_relation.c support routines for tables and column handling
-parse_target.c handle the result list of the query
-parse_type.c support routines for data type handling
-parse_utilcmd.c parse analysis for utility commands (done at execution time)
+ parser.c things start here
+ scan.l break query into tokens
+ scansup.c handle escapes in input strings
+ gram.y parse the tokens and produce a "raw" parse tree
+ analyze.c top level of parse analysis for optimizable queries
+ parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
+ parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
+ parse_coerce.c handle coercing expressions to different data types
+ parse_collate.c assign collation information in completed expressions
+ parse_cte.c handle Common Table Expressions (WITH clauses)
+ parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
+ parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
+ parse_func.c handle functions, table.column and column identifiers
+ parse_merge.c handle MERGE
+ parse_node.c create nodes for various structures
+ parse_oper.c handle operators in expressions
+ parse_param.c handle Params (for the cases used in the core backend)
+ parse_relation.c support routines for tables and column handling
+ parse_target.c handle the result list of the query
+ parse_type.c support routines for data type handling
+ parse_utilcmd.c parse analysis for utility commands (done at execution time)
See also src/common/keywords.c, which contains the table of standard
keywords and the keyword lookup function. We separated that out because
diff --git a/src/backend/regex/README.md b/src/backend/regex/README.md
index 930d8ced0d..43b965e9cc 100644
--- a/src/backend/regex/README.md
+++ b/src/backend/regex/README.md
@@ -9,11 +9,13 @@ General source-file layout
There are six separately-compilable source files, five of which expose
exactly one exported function apiece:
+
regcomp.c: pg_regcomp
regexec.c: pg_regexec
regerror.c: pg_regerror
regfree.c: pg_regfree
regprefix.c: pg_regprefix
+
(The pg_ prefixes were added by the Postgres project to distinguish this
library version from any similar one that might be present on a particular
system. They'd need to be removed or replaced in any standalone version
@@ -22,8 +24,8 @@ of the library.)
The sixth file, regexport.c, exposes multiple functions that allow extraction
of info about a compiled regex (see regexport.h).
-There are additional source files regc_*.c that are #include'd in regcomp,
-and similarly additional source files rege_*.c that are #include'd in
+There are additional source files `regc_*.c` that are #include'd in regcomp,
+and similarly additional source files `rege_*.c` that are #include'd in
regexec. This was done to avoid exposing internal symbols globally;
all functions not meant to be part of the library API are static.
@@ -38,19 +40,19 @@ structs.)
What's where in src/backend/regex/:
-regcomp.c Top-level regex compilation code
-regc_color.c Color map management
-regc_cvec.c Character vector (cvec) management
-regc_lex.c Lexer
-regc_nfa.c NFA handling
-regc_locale.c Application-specific locale code from Tcl project
-regc_pg_locale.c Postgres-added application-specific locale code
-regexec.c Top-level regex execution code
-rege_dfa.c DFA creation and execution
-regerror.c pg_regerror: generate text for a regex error code
-regfree.c pg_regfree: API to free a no-longer-needed regex_t
-regexport.c Functions for extracting info from a regex_t
-regprefix.c Code for extracting a common prefix from a regex_t
+ regcomp.c Top-level regex compilation code
+ regc_color.c Color map management
+ regc_cvec.c Character vector (cvec) management
+ regc_lex.c Lexer
+ regc_nfa.c NFA handling
+ regc_locale.c Application-specific locale code from Tcl project
+ regc_pg_locale.c Postgres-added application-specific locale code
+ regexec.c Top-level regex execution code
+ rege_dfa.c DFA creation and execution
+ regerror.c pg_regerror: generate text for a regex error code
+ regfree.c pg_regfree: API to free a no-longer-needed regex_t
+ regexport.c Functions for extracting info from a regex_t
+ regprefix.c Code for extracting a common prefix from a regex_t
The locale-specific code is concerned primarily with case-folding and with
expanding locale-specific character classes, such as [[:alnum:]]. It
@@ -58,11 +60,11 @@ really needs refactoring if this is ever to become a standalone library.
The header files for the library are in src/include/regex/:
-regcustom.h Customizes library for particular application
-regerrs.h Error message list
-regex.h Exported API
-regexport.h Exported API for regexport.c
-regguts.h Internals declarations
+ regcustom.h Customizes library for particular application
+ regerrs.h Error message list
+ regex.h Exported API
+ regexport.h Exported API for regexport.c
+ regguts.h Internals declarations
DFAs, NFAs, and all that
@@ -261,15 +263,19 @@ an additional arc labeled 2 wherever there is an arc labeled 3; this
action ensures that characters of color 2 (i.e., "x") will still be
considered as allowing any transitions they did before. We are now done
parsing the regex, and we have these final color assignments:
+
color 1: "a"
color 2: "x"
color 3: other letters
color 4: digits
+
and the NFA has these arcs:
+
states 1 -> 2 on color 1 (hence, "a" only)
states 2 -> 3 on color 4 (digits)
states 3 -> 4 on colors 1, 3, 4, and 2 (covering all \w characters)
states 4 -> 5 on color 2 ("x" only)
+
which can be seen to be a correct representation of the regex.
There is one more complexity, which is how to handle ".", that is a
diff --git a/src/backend/snowball/README.md b/src/backend/snowball/README.md
index 675baff5c9..ab4e4044d3 100644
--- a/src/backend/snowball/README.md
+++ b/src/backend/snowball/README.md
@@ -10,12 +10,12 @@ which is released by them under a BSD-style license.
The Snowball project does not often make formal releases; it's best
to pull from their git repository
-git clone https://github.com/snowballstem/snowball.git
+ git clone https://github.com/snowballstem/snowball.git
and then building the derived files is as simple as
-cd snowball
-make
+ cd snowball
+ make
At least on Linux, no platform-specific adjustment is needed.
@@ -39,10 +39,10 @@ To update the PostgreSQL sources from a new Snowball version:
1. Copy the *.c files in snowball/src_c/ to src/backend/snowball/libstemmer
with replacement of "../runtime/header.h" by "header.h", for example
-for f in .../snowball/src_c/*.c
-do
- sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
-done
+ for f in .../snowball/src_c/*.c
+ do
+ sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
+ done
Do not copy stemmers that are listed in libstemmer/modules.txt as
nonstandard, such as "german2" or "lovins".
diff --git a/src/backend/storage/freespace/README.md b/src/backend/storage/freespace/README.md
index dc2a63a137..6d8882c3da 100644
--- a/src/backend/storage/freespace/README.md
+++ b/src/backend/storage/freespace/README.md
@@ -33,9 +33,9 @@ node stores the max amount of free space on any of its children.
For example:
- 4
- 4 2
-3 4 0 2 <- This level represents heap pages
+ 4
+ 4 2
+ 3 4 0 2 <- This level represents heap pages
We need two basic operations: search and update.
@@ -67,10 +67,10 @@ header takes some space on a page, the binary tree isn't perfect. That is,
a few right-most leaf nodes are missing, and there are some useless non-leaf
nodes at the right. So the tree looks something like this:
- 0
- 1 2
- 3 4 5 6
-7 8 9 A B
+ 0
+ 1 2
+ 3 4 5 6
+ 7 8 9 A B
where the numbers denote each node's position in the array. Note that the
tree is guaranteed complete above the leaf level; only some leaf nodes are
@@ -100,27 +100,27 @@ For example, assuming each FSM page can hold information about 4 pages (in
reality, it holds (BLCKSZ - headers) / 2, or ~4000 with default BLCKSZ),
we get a disk layout like this:
- 0 <-- page 0 at level 2 (root page)
- 0 <-- page 0 at level 1
- 0 <-- page 0 at level 0
- 1 <-- page 1 at level 0
- 2 <-- ...
- 3
- 1 <-- page 1 at level 1
- 4
- 5
- 6
- 7
- 2
- 8
- 9
- 10
- 11
- 3
- 12
- 13
- 14
- 15
+ 0 <-- page 0 at level 2 (root page)
+ 0 <-- page 0 at level 1
+ 0 <-- page 0 at level 0
+ 1 <-- page 1 at level 0
+ 2 <-- ...
+ 3
+ 1 <-- page 1 at level 1
+ 4
+ 5
+ 6
+ 7
+ 2
+ 8
+ 9
+ 10
+ 11
+ 3
+ 12
+ 13
+ 14
+ 15
where the numbers are page numbers *at that level*, starting from 0.
diff --git a/src/backend/storage/lmgr/README-SSI.md b/src/backend/storage/lmgr/README-SSI.md
index 50d2ecca9d..76a4f40ce6 100644
--- a/src/backend/storage/lmgr/README-SSI.md
+++ b/src/backend/storage/lmgr/README-SSI.md
@@ -199,24 +199,24 @@ The PostgreSQL implementation uses two additional optimizations:
PostgreSQL Implementation
-------------------------
- * Since this technique is based on Snapshot Isolation (SI), those
+* Since this technique is based on Snapshot Isolation (SI), those
areas in PostgreSQL which don't use SI can't be brought under SSI.
This includes system tables, temporary tables, sequences, hint bit
rewrites, etc. SSI can not eliminate existing anomalies in these
areas.
- * Any transaction which is run at a transaction isolation level
+* Any transaction which is run at a transaction isolation level
other than SERIALIZABLE will not be affected by SSI. If you want to
enforce business rules through SSI, all transactions should be run at
the SERIALIZABLE transaction isolation level, and that should
probably be set as the default.
- * If all transactions are run at the SERIALIZABLE transaction
+* If all transactions are run at the SERIALIZABLE transaction
isolation level, business rules can be enforced in triggers or
application code without ever having a need to acquire an explicit
lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
- * Those who want to continue to use snapshot isolation without
+* Those who want to continue to use snapshot isolation without
the additional protections of SSI (and the associated costs of
enforcing those protections), can use the REPEATABLE READ transaction
isolation level. This level retains its legacy behavior, which
@@ -224,21 +224,21 @@ is identical to the old SERIALIZABLE implementation and fully
consistent with the standard's requirements for the REPEATABLE READ
transaction isolation level.
- * Performance under this SSI implementation will be significantly
+* Performance under this SSI implementation will be significantly
improved if transactions which don't modify permanent tables are
declared to be READ ONLY before they begin reading data.
- * Performance under SSI will tend to degrade more rapidly with a
+* Performance under SSI will tend to degrade more rapidly with a
large number of active database transactions than under less strict
isolation levels. Limiting the number of active transactions through
use of a connection pool or similar techniques may be necessary to
maintain good performance.
- * Any transaction which must be rolled back to prevent
+* Any transaction which must be rolled back to prevent
serialization anomalies will fail with SQLSTATE 40001, which has a
standard meaning of "serialization failure".
- * This SSI implementation makes an effort to choose the
+* This SSI implementation makes an effort to choose the
transaction to be canceled such that an immediate retry of the
transaction will not fail due to conflicts with exactly the same
transactions. Pursuant to this goal, no transaction is canceled
@@ -303,20 +303,20 @@ Heap locking
Predicate locks will be acquired for the heap based on the following:
- * For a table scan, the entire relation will be locked.
+* For a table scan, the entire relation will be locked.
- * Each tuple read which is visible to the reading transaction
+* Each tuple read which is visible to the reading transaction
will be locked, whether or not it meets selection criteria; except
that there is no need to acquire an SIREAD lock on a tuple when the
transaction already holds a write lock on any tuple representing the
row, since a rw-conflict would also create a ww-dependency which
has more aggressive enforcement and thus will prevent any anomaly.
- * Modifying a heap tuple creates a rw-conflict with any transaction
+* Modifying a heap tuple creates a rw-conflict with any transaction
that holds a SIREAD lock on that tuple, or on the page or relation
that contains it.
- * Inserting a new tuple creates a rw-conflict with any transaction
+* Inserting a new tuple creates a rw-conflict with any transaction
holding a SIREAD lock on the entire relation. It doesn't conflict with
page-level locks, because page-level locks are only used to aggregate
tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
@@ -346,34 +346,34 @@ false positives, they should be minimized for performance reasons.
Several optimizations are possible, though not all are implemented yet:
- * An index scan which is just finding the right position for an
+* An index scan which is just finding the right position for an
index insertion or deletion need not acquire a predicate lock.
- * An index scan which is comparing for equality on the entire key
+* An index scan which is comparing for equality on the entire key
for a unique index need not acquire a predicate lock as long as a key
is found corresponding to a visible tuple which has not been modified
by another transaction -- there are no "between or around" gaps to
cover.
- * As long as built-in foreign key enforcement continues to use
+* As long as built-in foreign key enforcement continues to use
its current "special tricks" to deal with MVCC issues, predicate
locks should not be needed for scans done by enforcement code.
- * If a search determines that no rows can be found regardless of
+* If a search determines that no rows can be found regardless of
index contents because the search conditions are contradictory (e.g.,
x = 1 AND x = 2), then no predicate lock is needed.
Other index AM implementation considerations:
- * For an index AM that doesn't have support for predicate locking,
+* For an index AM that doesn't have support for predicate locking,
we just acquire a predicate lock on the whole index for any search.
- * B-tree index searches acquire predicate locks only on the
+* B-tree index searches acquire predicate locks only on the
index *leaf* pages needed to lock the appropriate index range. If,
however, a search discovers that no root page has yet been created, a
predicate lock on the index relation is required.
- * Like a B-tree, GIN searches acquire predicate locks only on the
+* Like a B-tree, GIN searches acquire predicate locks only on the
leaf pages of entry tree. When performing an equality scan, and an
entry has a posting tree, the posting tree root is locked instead, to
lock only that key value. However, fastupdate=on postpones the
@@ -382,7 +382,7 @@ into pending list. That makes us unable to detect r-w conflicts using
page-level locks. To cope with that, insertions to the pending list
conflict with all scans.
- * GiST searches can determine that there are no matches at any
+* GiST searches can determine that there are no matches at any
level of the index, so we acquire predicate lock at each index
level during a GiST search. An index insert at the leaf level can
then be trusted to ripple up to all levels and locations where
@@ -390,13 +390,13 @@ conflicting predicate locks may exist. In case there is a page split,
we need to copy predicate lock from the original page to all the new
pages.
- * Hash index searches acquire predicate locks on the primary
+* Hash index searches acquire predicate locks on the primary
page of a bucket. It acquires a lock on both the old and new buckets
for scans that happen concurrently with page splits. During a bucket
split, a predicate lock is copied from the primary page of an old
bucket to the primary page of a new bucket.
- * The effects of page splits, overflows, consolidations, and
+* The effects of page splits, overflows, consolidations, and
removals must be carefully reviewed to ensure that predicate locks
aren't "lost" during those operations, or kept with pages which could
get re-used for different parts of the index.
@@ -409,56 +409,56 @@ The PostgreSQL implementation of Serializable Snapshot Isolation
differs from what is described in the cited papers for several
reasons:
- 1. PostgreSQL didn't have any existing predicate locking. It had
+1. PostgreSQL didn't have any existing predicate locking. It had
to be added from scratch.
- 2. The existing in-memory lock structures were not suitable for
+2. The existing in-memory lock structures were not suitable for
tracking SIREAD locks.
- * In PostgreSQL, tuple level locks are not held in RAM for
+- In PostgreSQL, tuple level locks are not held in RAM for
any length of time; lock information is written to the tuples
involved in the transactions.
- * In PostgreSQL, existing lock structures have pointers to
+- In PostgreSQL, existing lock structures have pointers to
memory which is related to a session. SIREAD locks need to persist
past the end of the originating transaction and even the session
which ran it.
- * PostgreSQL needs to be able to tolerate a large number of
+- PostgreSQL needs to be able to tolerate a large number of
transactions executing while one long-running transaction stays open
-- the in-RAM techniques discussed in the papers wouldn't support
that.
- 3. Unlike the database products used for the prototypes described
+3. Unlike the database products used for the prototypes described
in the papers, PostgreSQL didn't already have a true serializable
isolation level distinct from snapshot isolation.
- 4. PostgreSQL supports subtransactions -- an issue not mentioned
+4. PostgreSQL supports subtransactions -- an issue not mentioned
in the papers.
- 5. PostgreSQL doesn't assign a transaction number to a database
+5. PostgreSQL doesn't assign a transaction number to a database
transaction until and unless necessary (normally, when the transaction
attempts to modify data).
- 6. PostgreSQL has pluggable data types with user-definable
+6. PostgreSQL has pluggable data types with user-definable
operators, as well as pluggable index types, not all of which are
based around data types which support ordering.
- 7. Some possible optimizations became apparent during development
+7. Some possible optimizations became apparent during development
and testing.
Differences from the implementation described in the papers are
listed below.
- * New structures needed to be created in shared memory to track
+* New structures needed to be created in shared memory to track
the proper information for serializable transactions and their SIREAD
locks.
- * Because PostgreSQL does not have the same concept of an "oldest
+* Because PostgreSQL does not have the same concept of an "oldest
transaction ID" for all serializable transactions as assumed in the
Cahill thesis, we track the oldest snapshot xmin among serializable
transactions, and a count of how many active transactions use that
xmin. When the count hits zero we find the new oldest xmin and run a
clean-up based on that.
- * Because reads in a subtransaction may cause that subtransaction
+* Because reads in a subtransaction may cause that subtransaction
to roll back, thereby affecting what is written by the top level
transaction, predicate locks must survive a subtransaction rollback.
As a consequence, all xid usage in SSI, including predicate locking,
@@ -466,7 +466,7 @@ is based on the top level xid. When looking at an xid that comes
from a tuple's xmin or xmax, for example, we always call
SubTransGetTopmostTransaction() before doing much else with it.
- * PostgreSQL does not use "update in place" with a rollback log
+* PostgreSQL does not use "update in place" with a rollback log
for its MVCC implementation. Where possible it uses "HOT" updates on
the same page (if there is room and no indexed value is changed).
For non-HOT updates the old tuple is expired in place and a new tuple
@@ -477,21 +477,21 @@ versions of the row, based on the following proof that any additional
serialization failures we would get from that would be false
positives:
- o If transaction T1 reads a row version (thus acquiring a
+ - If transaction T1 reads a row version (thus acquiring a
predicate lock on it) and a second transaction T2 updates that row
version (thus creating a rw-conflict graph edge from T1 to T2), must a
third transaction T3 which re-updates the new version of the row also
have a rw-conflict in from T1 to prevent anomalies? In other words,
does it matter whether we recognize the edge T1 -> T3?
- o If T1 has a conflict in, it certainly doesn't. Adding the
+ - If T1 has a conflict in, it certainly doesn't. Adding the
edge T1 -> T3 would create a dangerous structure, but we already had
one from the edge T1 -> T2, so we would have aborted something anyway.
(T2 has already committed, else T3 could not have updated its output;
but we would have aborted either T1 or T1's predecessor(s). Hence
no cycle involving T1 and T3 can survive.)
- o Now let's consider the case where T1 doesn't have a
+ - Now let's consider the case where T1 doesn't have a
rw-conflict in. If that's the case, for this edge T1 -> T3 to make a
difference, T3 must have a rw-conflict out that induces a cycle in the
dependency graph, i.e. a conflict out to some transaction preceding T1
@@ -499,41 +499,41 @@ in the graph. (A conflict out to T1 itself would be problematic too,
but that would mean T1 has a conflict in, the case we already
eliminated.)
- o So now we're trying to figure out if there can be an
+ - So now we're trying to figure out if there can be an
rw-conflict edge T3 -> T0, where T0 is some transaction that precedes
T1. For T0 to precede T1, there has to be some edge, or sequence of
edges, from T0 to T1. At least the last edge has to be a wr-dependency
or ww-dependency rather than a rw-conflict, because T1 doesn't have a
rw-conflict in. And that gives us enough information about the order
of transactions to see that T3 can't have a rw-conflict to T0:
- - T0 committed before T1 started (the wr/ww-dependency implies this)
- - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
- - T2 committed before T3 started (otherwise, T3 would get aborted
+ - T0 committed before T1 started (the wr/ww-dependency implies this)
+ - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
+ - T2 committed before T3 started (otherwise, T3 would get aborted
because of an update conflict)
- o That means T0 committed before T3 started, and therefore
+ - That means T0 committed before T3 started, and therefore
there can't be a rw-conflict from T3 to T0.
- o So in all cases, we don't need the T1 -> T3 edge to
+ - So in all cases, we don't need the T1 -> T3 edge to
recognize cycles. Therefore it's not necessary for T1's SIREAD lock
on the original tuple version to cover later versions as well.
- * Predicate locking in PostgreSQL starts at the tuple level
+* Predicate locking in PostgreSQL starts at the tuple level
when possible. Multiple fine-grained locks are promoted to a single
coarser-granularity lock as needed to avoid resource exhaustion. The
amount of memory used for these structures is configurable, to balance
RAM usage against SIREAD lock granularity.
- * Each backend keeps a process-local table of the locks it holds.
+* Each backend keeps a process-local table of the locks it holds.
To support granularity promotion decisions with low CPU and locking
overhead, this table also includes the coarser covering locks and the
number of finer-granularity locks they cover.
- * Conflicts are identified by looking for predicate locks
+* Conflicts are identified by looking for predicate locks
when tuples are written, and by looking at the MVCC information when
tuples are read. There is no matching between two RAM-based locks.
- * Because write locks are stored in the heap tuples rather than a
+* Because write locks are stored in the heap tuples rather than a
RAM-based lock table, the optimization described in the Cahill thesis
which eliminates an SIREAD lock where there is a write lock is
implemented by the following:
@@ -543,18 +543,18 @@ predicate locks, a tuple lock on the tuple being written is removed.
return quickly without doing anything if it is a tuple written by the
reading transaction.
- * Rather than using conflictIn and conflictOut pointers which use
+* Rather than using conflictIn and conflictOut pointers which use
NULL to indicate no conflict and a self-reference to indicate
multiple conflicts or conflicts with committed transactions, we use a
list of rw-conflicts. With the more complete information, false
positives are reduced and we have sufficient data for more aggressive
clean-up and other optimizations:
- o We can avoid ever rolling back a transaction until and
+ - We can avoid ever rolling back a transaction until and
unless there is a pivot where a transaction on the conflict *out*
side of the pivot committed before either of the other transactions.
- o We can avoid ever rolling back a transaction when the
+ - We can avoid ever rolling back a transaction when the
transaction on the conflict *in* side of the pivot is explicitly or
implicitly READ ONLY unless the transaction on the conflict *out*
side of the pivot committed before the READ ONLY transaction acquired
@@ -562,25 +562,25 @@ its snapshot. (An implicit READ ONLY transaction is one which
committed without writing, even though it was not explicitly declared
to be READ ONLY.)
- o We can more aggressively clean up conflicts, predicate
+ - We can more aggressively clean up conflicts, predicate
locks, and SSI transaction information.
- * We allow a READ ONLY transaction to "opt out" of SSI if there are
+* We allow a READ ONLY transaction to "opt out" of SSI if there are
no READ WRITE transactions which could cause the READ ONLY
transaction to ever become part of a "dangerous structure" of
overlapping transaction dependencies.
- * We allow the user to request that a READ ONLY transaction wait
+* We allow the user to request that a READ ONLY transaction wait
until the conditions are right for it to start in the "opt out" state
described above. We add a DEFERRABLE state to transactions, which is
specified and maintained in a way similar to READ ONLY. It is
ignored for transactions that are not SERIALIZABLE and READ ONLY.
- * When a transaction must be rolled back, we pick among the
+* When a transaction must be rolled back, we pick among the
active transactions such that an immediate retry will not fail again
on conflicts with the same transactions.
- * We use the PostgreSQL SLRU system to hold summarized
+* We use the PostgreSQL SLRU system to hold summarized
information about older committed transactions to put an upper bound
on RAM used. Beyond that limit, information spills to disk.
Performance can degrade in a pessimal situation, but it should be
@@ -594,7 +594,7 @@ R&D Issues
This is intended to be the place to record specific issues which need
more detailed review or analysis.
- * WAL file replay. While serializable implementations using S2PL
+* WAL file replay. While serializable implementations using S2PL
can guarantee that the write-ahead log contains commits in a sequence
consistent with some serial execution of serializable transactions,
SSI cannot make that guarantee. While the WAL replay is no less
@@ -606,18 +606,18 @@ essence, if we do nothing, WAL replay will be at snapshot isolation
even for serializable transactions. Is this OK? If not, how do we
address it?
- * External replication. Look at how this impacts external
+* External replication. Look at how this impacts external
replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
This is related to the "WAL file replay" issue.
- * UNIQUE btree search for equality on all columns. Since a search
+* UNIQUE btree search for equality on all columns. Since a search
of a UNIQUE index using equality tests on all columns will lock the
heap tuple if an entry is found, it appears that there is no need to
get a predicate lock on the index in that case. A predicate lock is
still needed for such a search if a matching index entry which points
to a visible tuple is not found.
- * Minimize touching of shared memory. Should lists in shared
+* Minimize touching of shared memory. Should lists in shared
memory push entries which have just been returned to the front of the
available list, so they will be popped back off soon and some memory
might never be touched, or should we keep adding returned items to
@@ -638,9 +638,9 @@ http://dx.doi.org/10.1145/1071610.1071615
Architecture of a Database System. Foundations and Trends(R) in
Databases Vol. 1, No. 2 (2007) 141-259.
http://db.cs.berkeley.edu/papers/fntdb07-architecture.pdf
- Of particular interest:
- * 6.1 A Note on ACID
- * 6.2 A Brief Review of Serializability
- * 6.3 Locking and Latching
- * 6.3.1 Transaction Isolation Levels
- * 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
+Of particular interest:
+* 6.1 A Note on ACID
+* 6.2 A Brief Review of Serializability
+* 6.3 Locking and Latching
+* 6.3.1 Transaction Isolation Levels
+* 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
diff --git a/src/backend/utils/fmgr/README.md b/src/backend/utils/fmgr/README.md
index 9958d38992..ee14362b8e 100644
--- a/src/backend/utils/fmgr/README.md
+++ b/src/backend/utils/fmgr/README.md
@@ -20,18 +20,18 @@ tuple.)
When a function is looked up in pg_proc, the result is represented as
-typedef struct
-{
- PGFunction fn_addr; /* pointer to function or handler to be called */
- Oid fn_oid; /* OID of function (NOT of handler, if any) */
- short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
- bool fn_strict; /* function is "strict" (NULL in => NULL out) */
- bool fn_retset; /* function returns a set (over multiple calls) */
- unsigned char fn_stats; /* collect stats if track_functions > this */
- void *fn_extra; /* extra space for use by handler */
- MemoryContext fn_mcxt; /* memory context to store fn_extra in */
- Node *fn_expr; /* expression parse tree for call, or NULL */
-} FmgrInfo;
+ typedef struct
+ {
+ PGFunction fn_addr; /* pointer to function or handler to be called */
+ Oid fn_oid; /* OID of function (NOT of handler, if any) */
+ short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
+ bool fn_strict; /* function is "strict" (NULL in => NULL out) */
+ bool fn_retset; /* function returns a set (over multiple calls) */
+ unsigned char fn_stats; /* collect stats if track_functions > this */
+ void *fn_extra; /* extra space for use by handler */
+ MemoryContext fn_mcxt; /* memory context to store fn_extra in */
+ Node *fn_expr; /* expression parse tree for call, or NULL */
+ } FmgrInfo;
For an ordinary built-in function, fn_addr is just the address of the C
routine that implements the function. Otherwise it is the address of a
@@ -59,17 +59,17 @@ FmgrInfo than in FunctionCallInfoBaseData where it might more logically go.
During a call of a function, the following data structure is created
and passed to the function:
-typedef struct
-{
- FmgrInfo *flinfo; /* ptr to lookup info used for this call */
- Node *context; /* pass info about context of call */
- Node *resultinfo; /* pass or return extra info about result */
- Oid fncollation; /* collation for function to use */
- bool isnull; /* function must set true if result is NULL */
- short nargs; /* # arguments actually passed */
- NullableDatum args[]; /* Arguments passed to function */
-} FunctionCallInfoBaseData;
-typedef FunctionCallInfoBaseData* FunctionCallInfo;
+ typedef struct
+ {
+ FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+ Node *context; /* pass info about context of call */
+ Node *resultinfo; /* pass or return extra info about result */
+ Oid fncollation; /* collation for function to use */
+ bool isnull; /* function must set true if result is NULL */
+ short nargs; /* # arguments actually passed */
+ NullableDatum args[]; /* Arguments passed to function */
+ } FunctionCallInfoBaseData;
+ typedef FunctionCallInfoBaseData* FunctionCallInfo;
flinfo points to the lookup info used to make the call. Ordinary functions
will probably ignore this field, but function class handlers will need it
@@ -119,11 +119,11 @@ least), and other uglinesses.
Callees, whether they be individual functions or function handlers,
shall always have this signature:
-Datum function (FunctionCallInfo fcinfo);
+ Datum function (FunctionCallInfo fcinfo);
which is represented by the typedef
-typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
+ typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
The function is responsible for setting fcinfo->isnull appropriately
as well as returning a result represented as a Datum. Note that since
@@ -139,11 +139,11 @@ Here are the proposed macros and coding conventions:
The definition of an fmgr-callable function will always look like
-Datum
-function_name(PG_FUNCTION_ARGS)
-{
- ...
-}
+ Datum
+ function_name(PG_FUNCTION_ARGS)
+ {
+ ...
+ }
"PG_FUNCTION_ARGS" just expands to "FunctionCallInfo fcinfo". The main
reason for using this macro is to make it easy for scripts to spot function
@@ -156,26 +156,37 @@ just "fcinfo->args[n].isnull"). It should avoid trying to fetch the value
of any argument that is null.
Both strict and nonstrict functions can return NULL, if needed, with
+
PG_RETURN_NULL();
+
which expands to
+
{ fcinfo->isnull = true; return (Datum) 0; }
Argument values are ordinarily fetched using code like
+
int32 name = PG_GETARG_INT32(number);
For float4, float8, and int8, the PG_GETARG macros will hide whether the
types are pass-by-value or pass-by-reference. For example, if float8 is
pass-by-reference then PG_GETARG_FLOAT8 expands to
+
+
(* (float8 *) DatumGetPointer(fcinfo->args[number].value))
+
and would typically be called like this:
+
float8 arg = PG_GETARG_FLOAT8(0);
+
For what are now historical reasons, the float-related typedefs and macros
express the type width in bytes (4 or 8), whereas we prefer to label the
widths of integer types in bits.
Non-null values are returned with a PG_RETURN_XXX macro of the appropriate
type. For example, PG_RETURN_INT32 expands to
+
return Int32GetDatum(x)
+
PG_RETURN_FLOAT4, PG_RETURN_FLOAT8, and PG_RETURN_INT64 hide whether their
data types are pass-by-value or pass-by-reference, by doing a palloc if
needed.
@@ -290,9 +301,13 @@ transaction cleanup. SQL-callable functions can support this need
using the ErrorSaveContext context mechanism.
To report a "soft" error, a SQL-callable function should call
+
errsave(fcinfo->context, ...)
+
where it would previously have done
+
ereport(ERROR, ...)
+
If the passed "context" is NULL or is not an ErrorSaveContext node,
then errsave behaves precisely as ereport(ERROR): the exception is
thrown via longjmp, so that control does not return. If "context"
@@ -306,7 +321,9 @@ been reported in the ErrorSaveContext node.)
If there is nothing to do except return after calling errsave(),
you can save a line or two by writing
+
ereturn(fcinfo->context, dummy_value, ...)
+
to perform errsave() and then "return dummy_value".
An error reported "softly" must be safe, in the sense that there is
diff --git a/src/backend/utils/mb/README.md b/src/backend/utils/mb/README.md
index ef36626891..f1299fa2fd 100644
--- a/src/backend/utils/mb/README.md
+++ b/src/backend/utils/mb/README.md
@@ -3,20 +3,20 @@ src/backend/utils/mb/README
Encodings
=========
-conv.c: static functions and a public table for code conversion
-mbutils.c: public functions for the backend only.
-stringinfo_mb.c: public backend-only multibyte-aware stringinfo functions
-wstrcmp.c: strcmp for mb
-wstrncmp.c: strncmp for mb
-win866.c: a tool to generate KOI8 <--> CP866 conversion table
-iso.c: a tool to generate KOI8 <--> ISO8859-5 conversion table
-win1251.c: a tool to generate KOI8 <--> CP1251 conversion table
+ conv.c: static functions and a public table for code conversion
+ mbutils.c: public functions for the backend only.
+ stringinfo_mb.c: public backend-only multibyte-aware stringinfo functions
+ wstrcmp.c: strcmp for mb
+ wstrncmp.c: strncmp for mb
+ win866.c: a tool to generate KOI8 <--> CP866 conversion table
+ iso.c: a tool to generate KOI8 <--> ISO8859-5 conversion table
+ win1251.c: a tool to generate KOI8 <--> CP1251 conversion table
See also in src/common/:
-encnames.c: public functions for encoding names
-wchar.c: mostly static functions and a public table for mb string and
- multibyte conversion
+ encnames.c: public functions for encoding names
+ wchar.c: mostly static functions and a public table for mb string and
+ multibyte conversion
Introduction
------------
diff --git a/src/backend/utils/misc/README.md b/src/backend/utils/misc/README.md
index 85d97d29b6..abfa473757 100644
--- a/src/backend/utils/misc/README.md
+++ b/src/backend/utils/misc/README.md
@@ -23,29 +23,37 @@ modify the default SHOW display for a variable.
If a check_hook is provided, it points to a function of the signature
+
bool check_hook(datatype *newvalue, void **extra, GucSource source)
+
The "newvalue" argument is of type bool *, int *, double *, or char **
for bool, int/enum, real, or string variables respectively. The check
function should validate the proposed new value, and return true if it is
OK or false if not. The function can optionally do a few other things:
* When rejecting a bad proposed value, it may be useful to append some
-additional information to the generic "invalid value for parameter FOO"
-complaint that guc.c will emit. To do that, call
+ additional information to the generic "invalid value for parameter FOO"
+ complaint that guc.c will emit. To do that, call
+
void GUC_check_errdetail(const char *format, ...)
-where the format string and additional arguments follow the rules for
-errdetail() arguments. The resulting string will be emitted as the
-DETAIL line of guc.c's error report, so it should follow the message style
-guidelines for DETAIL messages. There is also
+
+ where the format string and additional arguments follow the rules for
+ errdetail() arguments. The resulting string will be emitted as the
+ DETAIL line of guc.c's error report, so it should follow the message style
+ guidelines for DETAIL messages. There is also
+
void GUC_check_errhint(const char *format, ...)
-which can be used in the same way to append a HINT message.
-Occasionally it may even be appropriate to override guc.c's generic primary
-message or error code, which can be done with
+
+ which can be used in the same way to append a HINT message.
+ Occasionally it may even be appropriate to override guc.c's generic primary
+ message or error code, which can be done with
+
void GUC_check_errcode(int sqlerrcode)
void GUC_check_errmsg(const char *format, ...)
-In general, check_hooks should avoid throwing errors directly if possible,
-though this may be impractical to avoid for some corner cases such as
-out-of-memory.
+
+ In general, check_hooks should avoid throwing errors directly if possible,
+ though this may be impractical to avoid for some corner cases such as
+ out-of-memory.
* Since the newvalue is pass-by-reference, the function can modify it.
This might be used for example to canonicalize the spelling of a string
@@ -76,7 +84,9 @@ assignment will occur.
If an assign_hook is provided, it points to a function of the signature
+
void assign_hook(datatype newvalue, void *extra)
+
where the type of "newvalue" matches the kind of variable, and "extra"
is the derived-information pointer returned by the check_hook (always
NULL if there is no check_hook). This function is called immediately
@@ -110,7 +120,9 @@ needing to check GUC values outside a transaction.
If a show_hook is provided, it points to a function of the signature
+
const char *show_hook(void)
+
This hook allows variable-specific computation of the value displayed
by SHOW (and other SQL features for showing GUC variable values).
The return value can point to a static buffer, since show functions are
@@ -214,23 +226,23 @@ The merged entry will have level N-1 and prior = older prior, so easiest
to keep older entry and free newer. There are 12 possibilities since
we already handled level N state = SAVE:
-N-1 N
+ N-1 N
-SAVE SET discard top prior, set state SET
-SAVE LOCAL discard top prior, no change to stack entry
-SAVE SET+LOCAL discard top prior, copy masked, state S+L
+ SAVE SET discard top prior, set state SET
+ SAVE LOCAL discard top prior, no change to stack entry
+ SAVE SET+LOCAL discard top prior, copy masked, state S+L
-SET SET discard top prior, no change to stack entry
-SET LOCAL copy top prior to masked, state S+L
-SET SET+LOCAL discard top prior, copy masked, state S+L
+ SET SET discard top prior, no change to stack entry
+ SET LOCAL copy top prior to masked, state S+L
+ SET SET+LOCAL discard top prior, copy masked, state S+L
-LOCAL SET discard top prior, set state SET
-LOCAL LOCAL discard top prior, no change to stack entry
-LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ LOCAL SET discard top prior, set state SET
+ LOCAL LOCAL discard top prior, no change to stack entry
+ LOCAL SET+LOCAL discard top prior, copy masked, state S+L
-SET+LOCAL SET discard top prior and second masked, state SET
-SET+LOCAL LOCAL discard top prior, no change to stack entry
-SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ SET+LOCAL SET discard top prior and second masked, state SET
+ SET+LOCAL LOCAL discard top prior, no change to stack entry
+ SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
RESET is executed like a SET, but using the reset_val as the desired new
diff --git a/src/backend/utils/mmgr/README.md b/src/backend/utils/mmgr/README.md
index 695088bb66..27a49a9a36 100644
--- a/src/backend/utils/mmgr/README.md
+++ b/src/backend/utils/mmgr/README.md
@@ -412,11 +412,11 @@ GetMemoryChunkMethodID() and finding the corresponding MemoryContextMethods
in the mcxt_methods[] array. For convenience, the MCXT_METHOD() macro is
provided, making the code as simple as:
-void
-pfree(void *pointer)
-{
- MCXT_METHOD(pointer, free_p)(pointer);
-}
+ void
+ pfree(void *pointer)
+ {
+ MCXT_METHOD(pointer, free_p)(pointer);
+ }
All of the current memory contexts make use of the MemoryChunk header type
which is defined in memutils_memorychunk.h. This suits all of the existing
diff --git a/src/backend/utils/resowner/README.md b/src/backend/utils/resowner/README.md
index cbf34e0b56..d0b4eb1d72 100644
--- a/src/backend/utils/resowner/README.md
+++ b/src/backend/utils/resowner/README.md
@@ -83,13 +83,13 @@ references, to name a few examples.
To add a new kind of resource, define a ResourceOwnerDesc to describe it.
For example:
-static const ResourceOwnerDesc myresource_desc = {
- .name = "My fancy resource",
- .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
- .release_priority = RELEASE_PRIO_FIRST,
- .ReleaseResource = ReleaseMyResource,
- .DebugPrint = PrintMyResource
-};
+ static const ResourceOwnerDesc myresource_desc = {
+ .name = "My fancy resource",
+ .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+ .release_priority = RELEASE_PRIO_FIRST,
+ .ReleaseResource = ReleaseMyResource,
+ .DebugPrint = PrintMyResource
+ };
ResourceOwnerRemember() and ResourceOwnerForget() functions take a pointer
to that struct, along with a Datum to represent the resource. The meaning
@@ -139,28 +139,28 @@ within each phase.
For example, imagine that you have two ResourceOwners, parent and child,
as follows:
-Parent
- parent resource BEFORE_LOCKS priority 1
- parent resource BEFORE_LOCKS priority 2
- parent resource AFTER_LOCKS priority 10001
- parent resource AFTER_LOCKS priority 10002
- Child
- child resource BEFORE_LOCKS priority 1
- child resource BEFORE_LOCKS priority 2
- child resource AFTER_LOCKS priority 10001
- child resource AFTER_LOCKS priority 10002
+ Parent
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
+ Child
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
These resources would be released in the following order:
-child resource BEFORE_LOCKS priority 1
-child resource BEFORE_LOCKS priority 2
-parent resource BEFORE_LOCKS priority 1
-parent resource BEFORE_LOCKS priority 2
-(locks)
-child resource AFTER_LOCKS priority 10001
-child resource AFTER_LOCKS priority 10002
-parent resource AFTER_LOCKS priority 10001
-parent resource AFTER_LOCKS priority 10002
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ (locks)
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
To release all the resources, you need to call ResourceOwnerRelease() three
times, once for each phase. You may perform additional tasks between the
diff --git a/src/interfaces/ecpg/preproc/README.parser.md b/src/interfaces/ecpg/preproc/README.parser.md
index ddc3061d48..27194dd956 100644
--- a/src/interfaces/ecpg/preproc/README.parser.md
+++ b/src/interfaces/ecpg/preproc/README.parser.md
@@ -14,29 +14,38 @@ ECPG modifies and extends the core grammar in a way that
actions for grammar rules.
In "ecpg.addons", every modified rule follows this pattern:
+
ECPG: dumpedtokens postfix
+
where "dumpedtokens" is simply tokens from core gram.y's
rules concatenated together. e.g. if gram.y has this:
- ruleA: tokenA tokenB tokenC {...}
+
+ ruleA: tokenA tokenB tokenC {...}
+
then "dumpedtokens" is "ruleAtokenAtokenBtokenC".
"postfix" above can be:
-a) "block" - the automatic rule created by parse.pl is completely
- overridden, the code block has to be written completely as
- it were in a plain bison grammar
-b) "rule" - the automatic rule is extended on, so new syntaxes
- are accepted for "ruleA". E.g.:
+
+* "block" - the automatic rule created by parse.pl is completely
+ overridden, the code block has to be written completely as
+ it were in a plain bison grammar
+* "rule" - the automatic rule is extended on, so new syntaxes
+ are accepted for "ruleA". E.g.:
+
ECPG: ruleAtokenAtokenBtokenC rule
| tokenD tokenE { action_code; }
...
+
It will be substituted with:
+
ruleA: <original syntax forms and actions up to and including
"tokenA tokenB tokenC">
| tokenD tokenE { action_code; }
...
-c) "addon" - the automatic action for the rule (SQL syntax constructed
- from the tokens concatenated together) is prepended with a new
- action code part. This code part is written as is's already inside
- the { ... }
+
+* "addon" - the automatic action for the rule (SQL syntax constructed
+ from the tokens concatenated together) is prepended with a new
+ action code part. This code part is written as is's already inside
+ the { ... }
Multiple "addon" or "block" lines may appear together with the
new code block if the code block is common for those rules.
diff --git a/src/port/README.md b/src/port/README.md
index ed5c54a72f..e5aeed07b6 100644
--- a/src/port/README.md
+++ b/src/port/README.md
@@ -14,7 +14,7 @@ libraries. This is done by removing -lpgport from the link line:
# Need to recompile any libpgport object files
LIBS := $(filter-out -lpgport, $(LIBS))
-and adding infrastructure to recompile the object files:
+ and adding infrastructure to recompile the object files:
OBJS= execute.o typename.o descriptor.o data.o error.o prepare.o memory.o \
connect.o misc.o path.o exec.o \
diff --git a/src/test/isolation/README.md b/src/test/isolation/README.md
index 5818ca5003..471b0a5029 100644
--- a/src/test/isolation/README.md
+++ b/src/test/isolation/README.md
@@ -12,23 +12,33 @@ serializable isolation level; but tests for other sorts of concurrent
behaviors have been added as well.
You can run the tests against the current build tree by typing
+
make check
+
Alternatively, you can run against an existing installation by typing
+
make installcheck
+
(This will contact a server at the default port expected by libpq.
You can set PGPORT and so forth in your environment to control this.)
To run just specific test(s) against an installed server,
you can do something like
+
./pg_isolation_regress fk-contention fk-deadlock
+
(look into the specs/ subdirectory to see the available tests).
Certain tests require the server's max_prepared_transactions parameter to be
set to at least 3; therefore they are not run by default. To include them in
the test run, use
+
make check-prepared-txns
+
or
+
make installcheck-prepared-txns
+
after making sure the server configuration is correct (see TEMP_CONFIG
to adjust this in the "check" case).
@@ -64,7 +74,7 @@ that are to be run.
A test specification consists of four parts, in this order:
-setup { <SQL> }
+setup { < SQL > }
The given SQL block is executed once (per permutation) before running
the test. Create any test tables or other required objects here. This
@@ -74,13 +84,13 @@ setup { <SQL> }
and some statements such as VACUUM cannot be combined with others in such
a block.)
-teardown { <SQL> }
+teardown { < SQL > }
The teardown SQL block is executed once after the test is finished. Use
this to clean up in preparation for the next permutation, e.g dropping
any test tables created by setup. This part is optional.
-session <name>
+session < name >
There are normally several "session" parts in a spec file. Each
session is executed in its own connection. A session part consists
@@ -91,13 +101,13 @@ session <name>
Each step has the syntax
- step <name> { <SQL> }
+ step < name > { < SQL > }
- where <name> is a name identifying this step, and <SQL> is a SQL statement
+ where < name > is a name identifying this step, and < SQL > is a SQL statement
(or statements, separated by semicolons) that is executed in the step.
Step names must be unique across the whole spec file.
-permutation <step name> ...
+permutation < step name > ...
A permutation line specifies a list of steps that are run in that order.
Any number of permutation lines can appear. If no permutation lines are
@@ -116,10 +126,10 @@ whether you quote them or not. You must use quotes if you want to use
an isolation test keyword (such as "permutation") as a name.
A # character begins a comment, which extends to the end of the line.
-(This does not work inside <SQL> blocks, however. Use the usual SQL
+(This does not work inside < SQL > blocks, however. Use the usual SQL
comment conventions there.)
-There is no way to include a "}" character in an <SQL> block.
+There is no way to include a "}" character in an < SQL > block.
For each permutation of the session steps (whether these are manually
specified in the spec file, or automatically generated), the isolation
@@ -187,9 +197,9 @@ step has completed. (If the other step is used more than once in the
current permutation, this step cannot complete while any of those
instances is active.)
-A marker of the form "<other step name> notices <n>" (where <n> is a
+A marker of the form "< other step name > notices < n >" (where < n > is a
positive integer) indicates that this step may not be reported as
-completing until the other step's session has returned at least <n>
+completing until the other step's session has returned at least < n >
NOTICE messages, counting from when this step is launched. This is useful
for stabilizing cases where a step can return NOTICE messages before it
actually completes, and those messages must be synchronized with the
diff --git a/src/test/kerberos/README.md b/src/test/kerberos/README.md
index a048d442af..dc53747fd9 100644
--- a/src/test/kerberos/README.md
+++ b/src/test/kerberos/README.md
@@ -23,9 +23,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/dblink and contrib/postgres_fdw in addition to the core code.
Run
+
make check PG_TEST_EXTRA=kerberos
+
or
+
make installcheck PG_TEST_EXTRA=kerberos
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/locale/README.md b/src/test/locale/README.md
index e290e31480..eb6a7c6124 100644
--- a/src/test/locale/README.md
+++ b/src/test/locale/README.md
@@ -9,8 +9,11 @@ locale data. Then there are test-sort.pl and test-sort.py that test
collating.
To run a test for some locale run
+
make check-$locale
+
for example
+
make check-koi8-r
Currently, there are only tests for a few locales available. The script
diff --git a/src/test/modules/dummy_seclabel/README.md b/src/test/modules/dummy_seclabel/README.md
index a3fcbd7599..1a12fdaad2 100644
--- a/src/test/modules/dummy_seclabel/README.md
+++ b/src/test/modules/dummy_seclabel/README.md
@@ -18,13 +18,13 @@ Usage
Here's a simple example of usage:
-# postgresql.conf
-shared_preload_libraries = 'dummy_seclabel'
+ # postgresql.conf
+ shared_preload_libraries = 'dummy_seclabel'
-postgres=# CREATE TABLE t (a int, b text);
-CREATE TABLE
-postgres=# SECURITY LABEL ON TABLE t IS 'classified';
-SECURITY LABEL
+ postgres=# CREATE TABLE t (a int, b text);
+ CREATE TABLE
+ postgres=# SECURITY LABEL ON TABLE t IS 'classified';
+ SECURITY LABEL
The dummy_seclabel module provides only four hardcoded
labels: unclassified, classified,
diff --git a/src/test/modules/test_parser/README.md b/src/test/modules/test_parser/README.md
index 0a11ec85fb..735bfe9a00 100644
--- a/src/test/modules/test_parser/README.md
+++ b/src/test/modules/test_parser/README.md
@@ -5,12 +5,12 @@ a starting point for developing your own parser.
test_parser recognizes words separated by white space,
and returns just two token types:
-mydb=# SELECT * FROM ts_token_type('testparser');
- tokid | alias | description
--------+-------+---------------
- 3 | word | Word
- 12 | blank | Space symbols
-(2 rows)
+ mydb=# SELECT * FROM ts_token_type('testparser');
+ tokid | alias | description
+ -------+-------+---------------
+ 3 | word | Word
+ 12 | blank | Space symbols
+ (2 rows)
These token numbers have been chosen to be compatible with the default
parser's numbering. This allows us to use its headline()
@@ -24,38 +24,38 @@ parser testparser. It has no user-configurable parameters.
You can test the parser with, for example,
-mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
- tokid | token
--------+--------
- 3 | That's
- 12 |
- 3 | my
- 12 |
- 3 | first
- 12 |
- 3 | own
- 12 |
- 3 | parser
+ mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
+ tokid | token
+ -------+--------
+ 3 | That's
+ 12 |
+ 3 | my
+ 12 |
+ 3 | first
+ 12 |
+ 3 | own
+ 12 |
+ 3 | parser
Real-world use requires setting up a text search configuration
that uses the parser. For example,
-mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
-CREATE TEXT SEARCH CONFIGURATION
-
-mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
-mydb-# ADD MAPPING FOR word WITH english_stem;
-ALTER TEXT SEARCH CONFIGURATION
-
-mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
- to_tsvector
--------------------------------
- 'that':1 'first':3 'parser':5
-(1 row)
-
-mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
-mydb(# to_tsquery('testcfg', 'star'));
- ts_headline
------------------------------------------------------------------
- Supernovae <b>stars</b> are the brightest phenomena in galaxies
-(1 row)
+ mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
+ CREATE TEXT SEARCH CONFIGURATION
+
+ mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
+ mydb-# ADD MAPPING FOR word WITH english_stem;
+ ALTER TEXT SEARCH CONFIGURATION
+
+ mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
+ to_tsvector
+ -------------------------------
+ 'that':1 'first':3 'parser':5
+ (1 row)
+
+ mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
+ mydb(# to_tsquery('testcfg', 'star'));
+ ts_headline
+ -----------------------------------------------------------------
+ Supernovae <b>stars</b> are the brightest phenomena in galaxies
+ (1 row)
diff --git a/src/test/modules/test_regex/README.md b/src/test/modules/test_regex/README.md
index 3ef152d4e1..7da5e67a9c 100644
--- a/src/test/modules/test_regex/README.md
+++ b/src/test/modules/test_regex/README.md
@@ -5,7 +5,7 @@ aren't currently exposed at the SQL level by PostgreSQL.
Currently, one function is provided:
-test_regex(pattern text, string text, flags text) returns setof text[]
+ test_regex(pattern text, string text, flags text) returns setof text[]
Reports an error if the pattern is an invalid regex. Otherwise,
the first row of output contains the number of subexpressions,
diff --git a/src/test/modules/test_rls_hooks/README.md b/src/test/modules/test_rls_hooks/README.md
index c22e0d3fb4..b561bb0192 100644
--- a/src/test/modules/test_rls_hooks/README.md
+++ b/src/test/modules/test_rls_hooks/README.md
@@ -3,14 +3,14 @@ define additional policies to be used.
Functions
=========
-test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with OR.
-test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with AND.
diff --git a/src/test/modules/test_shm_mq/README.md b/src/test/modules/test_shm_mq/README.md
index 641407bee0..1df3d38bde 100644
--- a/src/test/modules/test_shm_mq/README.md
+++ b/src/test/modules/test_shm_mq/README.md
@@ -14,9 +14,9 @@ Functions
=========
-test_shm_mq(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1)
- RETURNS void
+ test_shm_mq(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1)
+ RETURNS void
This function sends and receives messages synchronously. The user
backend sends the provided message to the first background worker using
@@ -31,10 +31,10 @@ the user backend verifies that the message finally received matches the
one originally sent and throws an error if not.
-test_shm_mq_pipelined(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1,
- verify bool default true)
- RETURNS void
+ test_shm_mq_pipelined(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1,
+ verify bool default true)
+ RETURNS void
This function sends the same message multiple times, as specified by the
repeat count, to the first background worker using a queue of the given
diff --git a/src/test/recovery/README.md b/src/test/recovery/README.md
index 896df0ad05..c18fe7e21e 100644
--- a/src/test/recovery/README.md
+++ b/src/test/recovery/README.md
@@ -14,9 +14,13 @@ contrib/pg_prewarm, contrib/pg_stat_statements and contrib/test_decoding
in addition to the core code.
Run
- make check
+
+ make check
+
or
- make installcheck
+
+ make installcheck
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/ssl/README.md b/src/test/ssl/README.md
index 2101a466d2..957694c689 100644
--- a/src/test/ssl/README.md
+++ b/src/test/ssl/README.md
@@ -20,9 +20,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/sslinfo in addition to the core code.
Run
+
make check PG_TEST_EXTRA=ssl
+
or
+
make installcheck PG_TEST_EXTRA=ssl
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
@@ -39,49 +43,49 @@ Certificates
The test suite needs a set of public/private key pairs and certificates to
run:
-root_ca
+* root_ca
root CA, use to sign the server and client CA certificates.
-server_ca
+* server_ca
CA used to sign server certificates.
-client_ca
+* client_ca
CA used to sign client certificates.
-server-cn-only
+* server-cn-only
server-cn-and-alt-names
server-single-alt-name
server-multiple-alt-names
-server-no-names
+server-no-names:
server certificates, with small variations in the hostnames present
in the certificate. Signed by server_ca.
-server-password
+* server-password:
same as server-cn-only, but password-protected.
-client
+* client:
a client certificate, for user "ssltestuser". Signed by client_ca.
-client-revoked
+* client-revoked:
like "client", but marked as revoked in the client CA's CRL.
In addition, there are a few files that combine various certificates together
in the same file:
-both-cas-1
+* both-cas-1:
Contains root_ca.crt, client_ca.crt and server_ca.crt, in that order.
-both-cas-2
+* both-cas-2:
Contains root_ca.crt, server_ca.crt and client_ca.crt, in that order.
-root+server_ca
+* root+server_ca:
Contains root_crt and server_ca.crt. For use as client's "sslrootcert"
option.
-root+client_ca
+* root+client_ca:
Contains root_crt and client_ca.crt. For use as server's "ssl_ca_file".
-client+client_ca
+* client+client_ca:
Contains client.crt and client_ca.crt in that order. For use as client's
certificate chain.
diff --git a/src/timezone/README.md b/src/timezone/README.md
index dd5d5f9892..811cddd2f7 100644
--- a/src/timezone/README.md
+++ b/src/timezone/README.md
@@ -79,15 +79,15 @@ fixed that.)
includes relying on configure's results rather than hand-hacked
#defines (see private.h in particular).
-* Similarly, avoid relying on <stdint.h> features that may not exist on old
+* Similarly, avoid relying on stdint.h features that may not exist on old
systems. In particular this means using Postgres' definitions of the int32
and int64 typedefs, not int_fast32_t/int_fast64_t. Likewise we use
PG_INT32_MIN/MAX not INT32_MIN/MAX. (Once we desupport all PG versions
-that don't require C99, it'd be practical to rely on <stdint.h> and remove
+that don't require C99, it'd be practical to rely on stdint.h and remove
this set of diffs; but that day is not yet.)
* Since Postgres is typically built on a system that has its own copy
-of the <time.h> functions, we must avoid conflicting with those. This
+of the time.h functions, we must avoid conflicting with those. This
mandates renaming typedef time_t to pg_time_t, and similarly for most
other exposed names.
diff --git a/src/timezone/tznames/README.md b/src/timezone/tznames/README.md
index 6d355e4616..55b98a3143 100644
--- a/src/timezone/tznames/README.md
+++ b/src/timezone/tznames/README.md
@@ -5,14 +5,14 @@ tznames
This directory contains files with timezone sets for PostgreSQL. The problem
is that time zone abbreviations are not unique throughout the world and you
-might find out that a time zone abbreviation in the `Default' set collides
+might find out that a time zone abbreviation in the 'Default' set collides
with the one you wanted to use. This can be fixed by selecting a timezone
set that defines the abbreviation the way you want it. There might already
be a file here that serves your needs. If not, you can create your own.
In order to use one of these files, you need to set
- timezone_abbreviations = 'xyz'
+ timezone_abbreviations = 'xyz'
in any of the usual ways for setting a parameter, where xyz is the filename
that contains the desired time zone abbreviations.
@@ -22,9 +22,9 @@ location supplied here, please report this to <[email protected]
Your set of time zone abbreviations can then be included in future releases.
For the time being you can always add your own set.
-Typically a custom abbreviation set is made by including the `Default' set
+Typically a custom abbreviation set is made by including the 'Default' set
and then adding or overriding abbreviations as necessary. For examples,
-see the `Australia' and `India' files.
+see the 'Australia' and 'India' files.
The files named Africa.txt, etc, are not intended to be used directly as
time zone abbreviation files. They contain reference definitions of time zone
diff --git a/src/tools/ci/README.md b/src/tools/ci/README.md
index 30ddd200c9..e82073ff8a 100644
--- a/src/tools/ci/README.md
+++ b/src/tools/ci/README.md
@@ -35,7 +35,7 @@ See also https://cirrus-ci.org/guide/quick-start/
Once enabled on a repository, future commits and pull-requests in that
repository will automatically trigger CI builds. These are visible from the
commit history / PRs, and can also be viewed in the cirrus-ci UI at
-https://cirrus-ci.com/github/<username>/<reponame>/
+https://cirrus-ci.com/github/< username >/< reponame >/
Hint: all build log files are uploaded to cirrus-ci and can be downloaded
from the "Artifacts" section from the cirrus-ci UI after clicking into a
@@ -74,7 +74,7 @@ When running a lot of tests in a repository, cirrus-ci's free credits do not
suffice. In those cases a repository can be configured to use other
infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
variable can be configured for the repository in the cirrus-ci web interface,
-at https://cirrus-ci.com/github/<user or organization>. The file referenced
+at https://cirrus-ci.com/github/< user or organization >. The file referenced
(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
overwrite the default execution method for different operating systems,
defined in .cirrus.yml, by redefining the relevant yaml anchors.
diff --git a/src/tools/pg_bsd_indent/README.md b/src/tools/pg_bsd_indent/README.md
index 992d4fce61..f1cb900dce 100644
--- a/src/tools/pg_bsd_indent/README.md
+++ b/src/tools/pg_bsd_indent/README.md
@@ -72,7 +72,7 @@ University of California, Berkeley
What follows is the README file as maintained by FreeBSD indent.
----------
-
+```
$FreeBSD: head/usr.bin/indent/README 105244 2002-10-16 13:58:39Z charnier $
This is the C indenter, it originally came from the University of Illinois
@@ -171,4 +171,4 @@ regards.. oz
cc: ccvaxa!willcox
sun.com!jar
uunet!rsalz
-
+```
diff --git a/src/tools/pgindent/README.md b/src/tools/pgindent/README.md
index b6cd4c6f6b..d984af8f1d 100644
--- a/src/tools/pgindent/README.md
+++ b/src/tools/pgindent/README.md
@@ -10,7 +10,7 @@ http://adpgtech.blogspot.com/2015/05/running-pgindent-on-non-core-code-or.html
PREREQUISITES:
-
+--------------
1) Install pg_bsd_indent in your PATH. Its source code is in the
sibling directory src/tools/pg_bsd_indent; see the directions
in that directory's README file.
@@ -22,13 +22,16 @@ PREREQUISITES:
To install, follow the usual install process for a Perl module
("man perlmodinstall" explains it). Or, if you have cpan installed,
this should work:
+
cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+
Or if you have cpanm installed, you can just use:
+
cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
-
+--------------------------------------------
1) Change directory to the top of the source tree.
2) Run pgindent on the C files:
@@ -64,7 +67,7 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
AT LEAST ONCE PER RELEASE CYCLE:
-
+--------------------------------
1) Download the latest typedef file from the buildfarm:
wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
--
2.39.3 (Apple Git-146)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-09-10 12:50 Daniel Gustafsson <[email protected]>
parent: Daniel Gustafsson <[email protected]>
0 siblings, 2 replies; 33+ messages in thread
From: Daniel Gustafsson @ 2024-09-10 12:50 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
> On 1 Jul 2024, at 12:22, Daniel Gustafsson <[email protected]> wrote:
> Attached is a v2 which fixes a conflict, if there is no interest in Markdown
> I'll drop 0001 and the markdown-specifics from 0002 to instead target increased
> consistency.
Since there doesn't seem to be much interest in going all the way to Markdown,
the attached 0001 is just the formatting changes for achieving (to some degree)
consistency among the README's. This mostly boils down to using a consistent
amount of whitespace around code, using the same indentation on bullet lists
and starting sections the same way. Inspecting the patch with git diff -w
reveals that it's not much left once whitespace is ignored. There might be a
few markdown hunks left which I'll hunt down in case anyone is interested in
this.
As an added bonus this still makes most READMEs render nicely as Markdown, just
not automatically on Github as it doesn't know the filetype.
--
Daniel Gustafsson
Attachments:
[application/octet-stream] v3-0001-Standardize-syntax-in-internal-documentation.patch (140.9K, ../../[email protected]/2-v3-0001-Standardize-syntax-in-internal-documentation.patch)
download | inline diff:
From f1ab59f7b5bdbdf90b25c201f4ab613c177381cb Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <[email protected]>
Date: Tue, 10 Sep 2024 14:23:47 +0200
Subject: [PATCH v3] Standardize syntax in internal documentation
This patchset intends to achieve a greater degree of consistency
regarding the use of formatting whitespace and bullet list chars
and section indicators in the README files. As an added benefit
this will make most of the files render as Markdown, some syntax
violations remain as we aren't targeting a Markdown conversion.
Reviewed-by: Erik Wienhold <[email protected]>
Reviewed-by: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
contrib/start-scripts/macos/README | 9 +-
src/backend/access/gin/README | 164 ++++---
src/backend/access/gist/README | 148 +++---
src/backend/access/hash/README | 76 +--
src/backend/access/heap/README.tuplock | 10 +-
src/backend/access/spgist/README | 80 +--
src/backend/access/transam/README | 56 +--
src/backend/lib/README | 22 +-
src/backend/libpq/README.SSL | 86 ++--
src/backend/optimizer/README | 244 ++++-----
src/backend/optimizer/plan/README | 106 ++--
src/backend/parser/README | 42 +-
src/backend/regex/README | 52 +-
src/backend/snowball/README | 14 +-
src/backend/storage/freespace/README | 56 +--
src/backend/storage/lmgr/README-SSI | 570 +++++++++++-----------
src/backend/utils/fmgr/README | 77 +--
src/backend/utils/mb/README | 16 +-
src/backend/utils/misc/README | 62 ++-
src/backend/utils/mmgr/README | 10 +-
src/backend/utils/resowner/README | 52 +-
src/interfaces/ecpg/preproc/README.parser | 23 +-
src/port/README | 2 +-
src/test/isolation/README | 30 +-
src/test/kerberos/README | 4 +
src/test/locale/README | 3 +
src/test/modules/dummy_seclabel/README | 12 +-
src/test/modules/test_parser/README | 74 +--
src/test/modules/test_regex/README | 2 +-
src/test/modules/test_rls_hooks/README | 8 +-
src/test/modules/test_shm_mq/README | 14 +-
src/test/recovery/README | 8 +-
src/test/ssl/README | 30 +-
src/timezone/tznames/README | 2 +-
src/tools/ci/README | 4 +-
src/tools/pgindent/README | 9 +-
36 files changed, 1134 insertions(+), 1043 deletions(-)
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README
index c4f2d9a270..8fe6efb657 100644
--- a/contrib/start-scripts/macos/README
+++ b/contrib/start-scripts/macos/README
@@ -15,10 +15,13 @@ if you plan to run the Postgres server under some user name other
than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
-/Library/LaunchDaemons/. You must do this as root:
- sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
-because the file will be ignored if it is not root-owned.
+ /Library/LaunchDaemons/. You must do this as root:
+
+ sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
+
+ because the file will be ignored if it is not root-owned.
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
+
sudo launchctl load /Library/LaunchDaemons/org.postgresql.postgres.plist
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README
index b080731621..08faa94410 100644
--- a/src/backend/access/gin/README
+++ b/src/backend/access/gin/README
@@ -40,7 +40,7 @@ Core PostgreSQL includes built-in Gin support for one-dimensional arrays
Synopsis
--------
-=# create index txt_idx on aa using gin(a);
+ =# create index txt_idx on aa using gin(a);
Features
--------
@@ -111,40 +111,42 @@ by building a "normal" index tuple and then modifying it.) The points to
know are:
* In a single-column index, a key tuple just contains the key datum, but
-in a multi-column index, a key tuple contains the pair (column number,
-key datum) where the column number is stored as an int2. This is needed
-to support different key data types in different columns. This much of
-the tuple is built by index_form_tuple according to the usual rules.
-The column number (if present) can never be null, but the key datum can
-be, in which case a null bitmap is present as usual. (As usual for index
-tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
+ in a multi-column index, a key tuple contains the pair (column number,
+ key datum) where the column number is stored as an int2. This is needed
+ to support different key data types in different columns. This much of
+ the tuple is built by index_form_tuple according to the usual rules.
+ The column number (if present) can never be null, but the key datum can
+ be, in which case a null bitmap is present as usual. (As usual for index
+ tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
* If the key datum is null (ie, IndexTupleHasNulls() is true), then
-just after the nominal index data (ie, at offset IndexInfoFindDataOffset
-or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
-the "category" of the null entry. These are the possible categories:
+ just after the nominal index data (ie, at offset IndexInfoFindDataOffset
+ or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
+ the "category" of the null entry. These are the possible categories:
+
1 = ordinary null key value extracted from an indexable item
2 = placeholder for zero-key indexable item
3 = placeholder for null indexable item
-Placeholder null entries are inserted into the index because otherwise
-there would be no index entry at all for an empty or null indexable item,
-which would mean that full index scans couldn't be done and various corner
-cases would give wrong answers. The different categories of null entries
-are treated as distinct keys by the btree, but heap itempointers for the
-same category of null entry are merged into one index entry just as happens
-with ordinary key entries.
+
+ Placeholder null entries are inserted into the index because otherwise
+ there would be no index entry at all for an empty or null indexable item,
+ which would mean that full index scans couldn't be done and various corner
+ cases would give wrong answers. The different categories of null entries
+ are treated as distinct keys by the btree, but heap itempointers for the
+ same category of null entry are merged into one index entry just as happens
+ with ordinary key entries.
* In a key entry at the btree leaf level, at the next SHORTALIGN boundary,
-there is a list of item pointers, in compressed format (see Posting List
-Compression section), pointing to the heap tuples for which the indexable
-items contain this key. This is called the "posting list".
-
-If the list would be too big for the index tuple to fit on an index page, the
-ItemPointers are pushed out to a separate posting page or pages, and none
-appear in the key entry itself. The separate pages are called a "posting
-tree" (see below); Note that in either case, the ItemPointers associated with
-a key can easily be read out in sorted order; this is relied on by the scan
-algorithms.
+ there is a list of item pointers, in compressed format (see Posting List
+ Compression section), pointing to the heap tuples for which the indexable
+ items contain this key. This is called the "posting list".
+
+ If the list would be too big for the index tuple to fit on an index page, the
+ ItemPointers are pushed out to a separate posting page or pages, and none
+ appear in the key entry itself. The separate pages are called a "posting
+ tree" (see below); Note that in either case, the ItemPointers associated with
+ a key can easily be read out in sorted order; this is relied on by the scan
+ algorithms.
* The index tuple header fields of a leaf key entry are abused as follows:
@@ -325,11 +327,11 @@ getting them on the next page.
The picture below shows tree state after finding the leaf page. Lower case
letters depicts tree pages. 'S' depicts shared lock on the page.
- a
- / | \
- b c d
- / | \ | \ | \
- eS f g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS f g h i j k
### Steping right
@@ -346,11 +348,11 @@ concurrently and doesn't delete right sibling accordingly.
The picture below shows two pages locked at once during stepping right.
- a
- / | \
- b c d
- / | \ | \ | \
- eS fS g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS fS g h i j k
### Insert
@@ -365,11 +367,11 @@ The picture below shows leaf page locked in exclusive mode and ready for
insertion. 'P' and 'E' depict pin and exclusive lock correspondingly.
- aP
- / | \
- b cP d
- / | \ | \ | \
- e f g hE i j k
+ aP
+ / | \
+ b cP d
+ / | \ | \ | \
+ e f g hE i j k
If insert causes a page split, the parent is locked in exclusive mode before
@@ -379,11 +381,11 @@ parent and child pages at once starting from child.
The picture below shows tree state after leaf page split. 'q' is new page
produced by split. Parent 'c' is about to have downlink inserted.
- aP
- / | \
- b cE d
- / | \ / | \ | \
- e f g hE q i j k
+ aP
+ / | \
+ b cE d
+ / | \ / | \ | \
+ e f g hE q i j k
### Page deletion
@@ -404,11 +406,11 @@ we locked it.
The picture below shows tree state after page deletion algorithm traversed to
leftmost leaf of the tree.
- aE
- / | \
- bE c d
- / | \ | \ | \
- eE f g h i j k
+ aE
+ / | \
+ bE c d
+ / | \ | \ | \
+ eE f g h i j k
Deletion algorithm keeps exclusive locks on left siblings of pages comprising
currently investigated path. Thus, if current page is to be removed, all
@@ -436,21 +438,21 @@ The picture below shows tree state after page deletion algorithm further
traversed the tree. Currently investigated path is 'a-c-h'. Left siblings 'b'
and 'g' of 'c' and 'h' correspondingly are also exclusively locked.
- aE
- / | \
- bE cE d
- / | \ | \ | \
- e f gE hE i j k
+ aE
+ / | \
+ bE cE d
+ / | \ | \ | \
+ e f gE hE i j k
The next picture shows tree state after page 'h' was deleted. It's marked with
'deleted' flag and newest xid, which might visit it. Downlink from 'c' to 'h'
is also deleted.
- aE
- / | \
- bE cE d
- / | \ \ | \
- e f gE hD iE j k
+ aE
+ / | \
+ bE cE d
+ / | \ \ | \
+ e f gE hD iE j k
However, it's still possible that concurrent reader has seen downlink from 'c'
to 'h' before we deleted it. In that case this reader will step right from 'h'
@@ -463,11 +465,11 @@ The next picture shows tree state after 'i' and 'c' was deleted. Internal page
investigation is 'a-d-j'. Pages 'b' and 'g' are locked as self siblings of 'd'
and 'j'.
- aE
- / \
- bE cD dE
- / | \ | \
- e f gE hD iD jE k
+ aE
+ / \
+ bE cD dE
+ / | \ | \
+ e f gE hD iD jE k
During the replay of page deletion at standby, the page's left sibling, the
target page, and its parent, are locked in that order. This order guarantees
@@ -516,44 +518,44 @@ For compatibility, old uncompressed format is also supported. Following
rules are used to handle it:
* GIN_ITUP_COMPRESSED flag marks index tuples that contain a posting list.
-This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
-Use GinItupIsCompressed(itup) to check the flag.
+ This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
+ Use GinItupIsCompressed(itup) to check the flag.
* Posting tree pages in the new format are marked with the GIN_COMPRESSED flag.
Macros GinPageIsCompressed(page) and GinPageSetCompressed(page) are used to
check and set this flag.
* All scan operations check format of posting list add use corresponding code
-to read its content.
+ to read its content.
* When updating an index tuple containing an uncompressed posting list, it
-will be replaced with new index tuple containing a compressed list.
+ will be replaced with new index tuple containing a compressed list.
* When updating an uncompressed posting tree leaf page, it's compressed.
* If vacuum finds some dead TIDs in uncompressed posting lists, they are
-converted into compressed posting lists. This assumes that the compressed
-posting list fits in the space occupied by the uncompressed list. IOW, we
-assume that the compressed version of the page, with the dead items removed,
-takes less space than the old uncompressed version.
+ converted into compressed posting lists. This assumes that the compressed
+ posting list fits in the space occupied by the uncompressed list. IOW, we
+ assume that the compressed version of the page, with the dead items removed,
+ takes less space than the old uncompressed version.
Limitations
-----------
- * Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
- * Gin searches entries only by equality matching, or simple range
- matching using the "partial match" feature.
+* Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
+* Gin searches entries only by equality matching, or simple range
+ matching using the "partial match" feature.
TODO
----
Nearest future:
- * Opclasses for more types (no programming, just many catalog changes)
+* Opclasses for more types (no programming, just many catalog changes)
Distant future:
- * Replace B-tree of entries to something like GiST
+* Replace B-tree of entries to something like GiST
Authors
-------
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README
index 8015ff19f0..af082fc2bb 100644
--- a/src/backend/access/gist/README
+++ b/src/backend/access/gist/README
@@ -183,70 +183,70 @@ operation.
findPath is a subroutine of findParent, used when the correct parent page
can't be found by following the rightlinks at the parent level:
-findPath( stack item )
- push stack, [root, 0, 0] // page, LSN, parent
- while( stack )
- ptr = top of stack
- latch( ptr->page, S-mode )
- if ( ptr->parent->page->lsn < ptr->page->nsn )
- push stack, [ ptr->page->rightlink, 0, ptr->parent ]
- end
- for( each tuple on page )
- if ( tuple->pagepointer == item->page )
- return stack
- else
- add to stack at the end [tuple->pagepointer,0, ptr]
+ findPath( stack item )
+ push stack, [root, 0, 0] // page, LSN, parent
+ while( stack )
+ ptr = top of stack
+ latch( ptr->page, S-mode )
+ if ( ptr->parent->page->lsn < ptr->page->nsn )
+ push stack, [ ptr->page->rightlink, 0, ptr->parent ]
+ end
+ for( each tuple on page )
+ if ( tuple->pagepointer == item->page )
+ return stack
+ else
+ add to stack at the end [tuple->pagepointer,0, ptr]
+ end
end
+ unlatch( ptr->page )
+ pop stack
end
- unlatch( ptr->page )
- pop stack
- end
gistFindCorrectParent is used to re-find the parent of a page during
insertion. It might have migrated to the right since we traversed down the
tree because of page splits.
-findParent( stack item )
- parent = item->parent
- if ( parent->page->lsn != parent->lsn )
- while(true)
- search parent tuple on parent->page, if found the return
- rightlink = parent->page->rightlink
- unlatch( parent->page )
- if ( rightlink is incorrect )
- break loop
+ findParent( stack item )
+ parent = item->parent
+ if ( parent->page->lsn != parent->lsn )
+ while(true)
+ search parent tuple on parent->page, if found the return
+ rightlink = parent->page->rightlink
+ unlatch( parent->page )
+ if ( rightlink is incorrect )
+ break loop
+ end
+ parent->page = rightlink
+ latch( parent->page, X-mode )
end
- parent->page = rightlink
+ newstack = findPath( item->parent )
+ replace part of stack to new one
latch( parent->page, X-mode )
+ return findParent( item )
end
- newstack = findPath( item->parent )
- replace part of stack to new one
- latch( parent->page, X-mode )
- return findParent( item )
- end
pageSplit function decides how to distribute keys to the new pages after
page split:
-pageSplit(page, allkeys)
- (lkeys, rkeys) = pickSplit( allkeys )
- if ( page is root )
- lpage = new page
- else
- lpage = page
- rpage = new page
- if ( no space left on rpage )
- newkeys = pageSplit( rpage, rkeys )
- else
- push newkeys, union(rkeys)
- end
- if ( no space left on lpage )
- push newkeys, pageSplit( lpage, lkeys )
- else
- push newkeys, union(lkeys)
- end
- return newkeys
+ pageSplit(page, allkeys)
+ (lkeys, rkeys) = pickSplit( allkeys )
+ if ( page is root )
+ lpage = new page
+ else
+ lpage = page
+ rpage = new page
+ if ( no space left on rpage )
+ newkeys = pageSplit( rpage, rkeys )
+ else
+ push newkeys, union(rkeys)
+ end
+ if ( no space left on lpage )
+ push newkeys, pageSplit( lpage, lkeys )
+ else
+ push newkeys, union(lkeys)
+ end
+ return newkeys
@@ -302,18 +302,18 @@ In the algorithm, levels are numbered so that leaf pages have level zero,
and internal node levels count up from 1. This numbering ensures that a page's
level number never changes, even when the root page is split.
-Level Tree
+ Level Tree
-3 *
- / \
-2 * *
- / | \ / | \
-1 * * * * * *
- / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o
+ 3 *
+ / \
+ 2 * *
+ / | \ / | \
+ 1 * * * * * *
+ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o
-* - internal page
-o - leaf page
+ * - internal page
+ o - leaf page
Internal pages that belong to certain levels have buffers associated with
them. Leaf pages never have buffers. Which levels have buffers is controlled
@@ -322,17 +322,17 @@ have buffers, while others do not. For example, if level_step = 2, then
pages on levels 2, 4, 6, ... have buffers. If level_step = 1 then every
internal page has a buffer.
-Level Tree (level_step = 1) Tree (level_step = 2)
+ Level Tree (level_step = 1) Tree (level_step = 2)
-3 * *
- / \ / \
-2 *(b) *(b) *(b) *(b)
- / | \ / | \ / | \ / | \
-1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
- / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o o o o o o o o o o o o o
+ 3 * *
+ / \ / \
+ 2 *(b) *(b) *(b) *(b)
+ / | \ / | \ / | \ / | \
+ 1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
+ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o o o o o o o o o o o o o
-(b) - buffer
+ (b) - buffer
Logically, a buffer is just bunch of tuples. Physically, it is divided in
pages, backed by a temporary file. Each buffer can be in one of two states:
@@ -365,7 +365,7 @@ pages where the tuples finally land on get cached too. If there are, the last
buffer page of each buffer below is kept in memory. This is illustrated in
the figures below:
- Buffer being emptied to
+ Buffer being emptied to
lower-level buffers Buffer being emptied to leaf pages
+(fb) +(fb)
@@ -374,11 +374,11 @@ the figures below:
/ \ / \ / \ / \
*(ab) *(ab) *(ab) *(ab) x x x x
-+ - cached internal page
-x - cached leaf page
-* - non-cached internal page
-(fb) - buffer being emptied
-(ab) - buffers being appended to, with last page in memory
+ + - cached internal page
+ x - cached leaf page
+ * - non-cached internal page
+ (fb) - buffer being emptied
+ (ab) - buffers being appended to, with last page in memory
In the beginning of the index build, the level-step is chosen so that all those
pages involved in emptying one buffer fit in cache, so after each of those
diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README
index 13dc59c124..a5df13a68a 100644
--- a/src/backend/access/hash/README
+++ b/src/backend/access/hash/README
@@ -248,24 +248,24 @@ track of available overflow pages.
The reader algorithm is:
- lock the primary bucket page of the target bucket
- if the target bucket is still being populated by a split:
- release the buffer content lock on current bucket page
- pin and acquire the buffer content lock on old bucket in shared mode
- release the buffer content lock on old bucket, but not pin
- retake the buffer content lock on new bucket
- arrange to scan the old bucket normally and the new bucket for
- tuples which are not moved-by-split
--- then, per read request:
- reacquire content lock on current page
- step to next page if necessary (no chaining of content locks, but keep
- the pin on the primary bucket throughout the scan)
- save all the matching tuples from current index page into an items array
- release pin and content lock (but if it is primary bucket page retain
- its pin till the end of the scan)
- get tuple from an item array
--- at scan shutdown:
- release all pins still held
+ lock the primary bucket page of the target bucket
+ if the target bucket is still being populated by a split:
+ release the buffer content lock on current bucket page
+ pin and acquire the buffer content lock on old bucket in shared mode
+ release the buffer content lock on old bucket, but not pin
+ retake the buffer content lock on new bucket
+ arrange to scan the old bucket normally and the new bucket for
+ tuples which are not moved-by-split
+ -- then, per read request:
+ reacquire content lock on current page
+ step to next page if necessary (no chaining of content locks, but keep
+ the pin on the primary bucket throughout the scan)
+ save all the matching tuples from current index page into an items array
+ release pin and content lock (but if it is primary bucket page retain
+ its pin till the end of the scan)
+ get tuple from an item array
+ -- at scan shutdown:
+ release all pins still held
Holding the buffer pin on the primary bucket page for the whole scan prevents
the reader's current-tuple pointer from being invalidated by splits or
@@ -288,8 +288,8 @@ which this bucket is formed by split.
The insertion algorithm is rather similar:
lock the primary bucket page of the target bucket
--- (so far same as reader, except for acquisition of buffer content lock in
- exclusive mode on primary bucket page)
+ -- (so far same as reader, except for acquisition of buffer content lock in
+ exclusive mode on primary bucket page)
if the bucket-being-split flag is set for a bucket and pin count on it is
one, then finish the split
release the buffer content lock on current bucket
@@ -465,24 +465,24 @@ overflow page to the free pool.
Obtaining an overflow page:
- take metapage content lock in exclusive mode
- determine next bitmap page number; if none, exit loop
- release meta page content lock
- pin bitmap page and take content lock in exclusive mode
- search for a free page (zero bit in bitmap)
- if found:
- set bit in bitmap
- mark bitmap page dirty
- take metapage buffer content lock in exclusive mode
- if first-free-bit value did not change,
- update it and mark meta page dirty
- else (not found):
- release bitmap page buffer content lock
- loop back to try next bitmap page, if any
--- here when we have checked all bitmap pages; we hold meta excl. lock
- extend index to add another overflow page; update meta information
- mark meta page dirty
- return page number
+ take metapage content lock in exclusive mode
+ determine next bitmap page number; if none, exit loop
+ release meta page content lock
+ pin bitmap page and take content lock in exclusive mode
+ search for a free page (zero bit in bitmap)
+ if found:
+ set bit in bitmap
+ mark bitmap page dirty
+ take metapage buffer content lock in exclusive mode
+ if first-free-bit value did not change,
+ update it and mark meta page dirty
+ else (not found):
+ release bitmap page buffer content lock
+ loop back to try next bitmap page, if any
+ -- here when we have checked all bitmap pages; we hold meta excl. lock
+ extend index to add another overflow page; update meta information
+ mark meta page dirty
+ return page number
It is slightly annoying to release and reacquire the metapage lock
multiple times, but it seems best to do it that way to minimize loss of
diff --git a/src/backend/access/heap/README.tuplock b/src/backend/access/heap/README.tuplock
index 6441e8baf0..5a91b4fddd 100644
--- a/src/backend/access/heap/README.tuplock
+++ b/src/backend/access/heap/README.tuplock
@@ -62,11 +62,11 @@ the tuple without changing its key.
The conflict table is:
- UPDATE NO KEY UPDATE SHARE KEY SHARE
-UPDATE conflict conflict conflict conflict
-NO KEY UPDATE conflict conflict conflict
-SHARE conflict conflict
-KEY SHARE conflict
+ UPDATE NO KEY UPDATE SHARE KEY SHARE
+ UPDATE conflict conflict conflict conflict
+ NO KEY UPDATE conflict conflict conflict
+ SHARE conflict conflict
+ KEY SHARE conflict
When there is a single locker in a tuple, we can just store the locking info
in the tuple itself. We do this by storing the locker's Xid in XMAX, and
diff --git a/src/backend/access/spgist/README b/src/backend/access/spgist/README
index 7117e02c77..2b890e4f8e 100644
--- a/src/backend/access/spgist/README
+++ b/src/backend/access/spgist/README
@@ -13,7 +13,7 @@ few disk pages, even if it traverses many nodes.
COMMON STRUCTURE DESCRIPTION
-
+----------------------------
Logically, an SP-GiST tree is a set of tuples, each of which can be either
an inner or leaf tuple. Each inner tuple contains "nodes", which are
(label,pointer) pairs, where the pointer (ItemPointerData) is a pointer to
@@ -58,27 +58,27 @@ pages.
An inner tuple consists of:
- optional prefix value - all successors must be consistent with it.
- Example:
- radix tree - prefix value is a common prefix string
- quad tree - centroid
- k-d tree - one coordinate
+ optional prefix value - all successors must be consistent with it.
+ Example:
+ radix tree - prefix value is a common prefix string
+ quad tree - centroid
+ k-d tree - one coordinate
- list of nodes, where node is a (label, pointer) pair.
- Example of a label: a single character for radix tree
+ list of nodes, where node is a (label, pointer) pair.
+ Example of a label: a single character for radix tree
A leaf tuple consists of:
- a leaf value
- Example:
- radix tree - the rest of string (postfix)
- quad and k-d tree - the point itself
+ a leaf value
+ Example:
+ radix tree - the rest of string (postfix)
+ quad and k-d tree - the point itself
- ItemPointer to the corresponding heap tuple
- nextOffset number of next leaf tuple in a chain on a leaf page
+ ItemPointer to the corresponding heap tuple
+ nextOffset number of next leaf tuple in a chain on a leaf page
- optional nulls bitmask
- optional INCLUDE-column values
+ optional nulls bitmask
+ optional INCLUDE-column values
For compatibility with pre-v14 indexes, a leaf tuple has a nulls bitmask
only if there are null values (among the leaf value and the INCLUDE values)
@@ -90,7 +90,7 @@ code can be used.
NULLS HANDLING
-
+--------------
We assume that SPGiST-indexable operators are strict (can never succeed for
null inputs). It is still desirable to index nulls, so that whole-table
indexscans are possible and so that "x IS NULL" can be implemented by an
@@ -104,27 +104,27 @@ AllTheSame cases in the normal tree.
INSERTION ALGORITHM
-
+-------------------
Insertion algorithm is designed to keep the tree in a consistent state at
any moment. Here is a simplified insertion algorithm specification
(numbers refer to notes below):
- Start with the first tuple on the root page (1)
-
- loop:
- if (page is leaf) then
- if (enough space)
- insert on page and exit (5)
- else (7)
- call PickSplitFn() (2)
- end if
- else
- switch (chooseFn())
- case MatchNode - descend through selected node
- case AddNode - add node and then retry chooseFn (3, 6)
- case SplitTuple - split inner tuple to prefix and postfix, then
- retry chooseFn with the prefix tuple (4, 6)
- end if
+ Start with the first tuple on the root page (1)
+
+ loop:
+ if (page is leaf) then
+ if (enough space)
+ insert on page and exit (5)
+ else (7)
+ call PickSplitFn() (2)
+ end if
+ else
+ switch (chooseFn())
+ case MatchNode - descend through selected node
+ case AddNode - add node and then retry chooseFn (3, 6)
+ case SplitTuple - split inner tuple to prefix and postfix, then
+ retry chooseFn with the prefix tuple (4, 6)
+ end if
Notes:
@@ -160,7 +160,7 @@ the following notation, where tuple's id is just for discussion (no such id
is actually stored):
inner tuple: {tuple id}(prefix string)[ comma separated list of node labels ]
-leaf tuple: {tuple id}<value>
+leaf tuple: {tuple id}< value >
Suppose we need to insert string 'www.gogo.com' into inner tuple
@@ -215,7 +215,7 @@ space utilization, but doesn't change the basis of the algorithm.
CONCURRENCY
-
+-----------
While descending the tree, the insertion algorithm holds exclusive lock on
two tree levels at a time, ie both parent and child pages (but parent and
child pages can be the same, see notes above). There is a possibility of
@@ -267,7 +267,7 @@ been flushed out of the system.
DEAD TUPLES
-
+-----------
Tuples on leaf pages can be in one of four states:
SPGIST_LIVE: normal, live pointer to a heap tuple.
@@ -319,7 +319,7 @@ remove unused inner tuples.
VACUUM
-
+------
VACUUM (or more precisely, spgbulkdelete) performs a single sequential scan
over the entire index. On both leaf and inner pages, we can convert old
REDIRECT tuples into PLACEHOLDER status, and then remove any PLACEHOLDERs
@@ -374,7 +374,7 @@ space map, and gather statistics.
LAST USED PAGE MANAGEMENT
-
+-------------------------
The list of last used pages contains four pages - a leaf page and three
inner pages, one from each "triple parity" group. (Actually, there's one
such list for the main tree and a separate one for the nulls tree.) This
@@ -384,6 +384,6 @@ critical, because we could allocate a new page at any moment.
AUTHORS
-
+-------
Teodor Sigaev <[email protected]>
Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README
index 6e4711dace..62015da1fe 100644
--- a/src/backend/access/transam/README
+++ b/src/backend/access/transam/README
@@ -59,27 +59,27 @@ For example, consider the following sequence of user commands:
In the main processing loop, this results in the following function call
sequence:
- / StartTransactionCommand;
- / StartTransaction;
-1) < ProcessUtility; << BEGIN
- \ BeginTransactionBlock;
- \ CommitTransactionCommand;
-
- / StartTransactionCommand;
-2) / PortalRunSelect; << SELECT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
-3) / ProcessQuery; << INSERT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
- / ProcessUtility; << COMMIT
-4) < EndTransactionBlock;
- \ CommitTransactionCommand;
- \ CommitTransaction;
+ / StartTransactionCommand;
+ / StartTransaction;
+ 1) < ProcessUtility; << BEGIN
+ \ BeginTransactionBlock;
+ \ CommitTransactionCommand;
+
+ / StartTransactionCommand;
+ 2) / PortalRunSelect; << SELECT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ 3) / ProcessQuery; << INSERT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ / ProcessUtility; << COMMIT
+ 4) < EndTransactionBlock;
+ \ CommitTransactionCommand;
+ \ CommitTransaction;
The point of this example is to demonstrate the need for
StartTransactionCommand and CommitTransactionCommand to be state smart -- they
@@ -100,12 +100,12 @@ Transaction aborts can occur in two ways:
The reason we have to distinguish them is illustrated by the following two
situations:
- case 1 case 2
- ------ ------
-1) user types BEGIN 1) user types BEGIN
-2) user does something 2) user does something
-3) user does not like what 3) system aborts for some reason
- she sees and types ABORT (syntax error, etc)
+ case 1 case 2
+ ------ ------
+ 1) user types BEGIN 1) user types BEGIN
+ 2) user does something 2) user does something
+ 3) user does not like what 3) system aborts for some reason
+ she sees and types ABORT (syntax error, etc)
In case 1, we want to abort the transaction and return to the default state.
In case 2, there may be more commands coming our way which are part of the
@@ -171,7 +171,7 @@ CommitTransactionCommand, the real work is done. The main point of doing
things this way is that if we get an error while popping state stack entries,
the remaining stack entries still show what we need to do to finish up.
-In the case of ROLLBACK TO <savepoint>, we abort all the subtransactions up
+In the case of ROLLBACK TO < savepoint >, we abort all the subtransactions up
through the one identified by the savepoint name, and then re-create that
subtransaction level with the same name. So it's a completely new
subtransaction as far as the internals are concerned.
diff --git a/src/backend/lib/README b/src/backend/lib/README
index f2fb591237..fc8e1aa1f7 100644
--- a/src/backend/lib/README
+++ b/src/backend/lib/README
@@ -1,27 +1,27 @@
This directory contains a general purpose data structures, for use anywhere
in the backend:
-binaryheap.c - a binary heap
+ binaryheap.c - a binary heap
-bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
+ bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
-bloomfilter.c - probabilistic, space-efficient set membership testing
+ bloomfilter.c - probabilistic, space-efficient set membership testing
-dshash.c - concurrent hash tables backed by dynamic shared memory areas
+ dshash.c - concurrent hash tables backed by dynamic shared memory areas
-hyperloglog.c - a streaming cardinality estimator
+ hyperloglog.c - a streaming cardinality estimator
-ilist.c - single and double-linked lists
+ ilist.c - single and double-linked lists
-integerset.c - a data structure for holding large set of integers
+ integerset.c - a data structure for holding large set of integers
-knapsack.c - knapsack problem solver
+ knapsack.c - knapsack problem solver
-pairingheap.c - a pairing heap
+ pairingheap.c - a pairing heap
-rbtree.c - a red-black tree
+ rbtree.c - a red-black tree
-stringinfo.c - an extensible string type
+ stringinfo.c - an extensible string type
Aside from the inherent characteristics of the data structures, there are a
diff --git a/src/backend/libpq/README.SSL b/src/backend/libpq/README.SSL
index d84a434a6e..98e20498bd 100644
--- a/src/backend/libpq/README.SSL
+++ b/src/backend/libpq/README.SSL
@@ -3,59 +3,59 @@ src/backend/libpq/README.SSL
SSL
===
->From the servers perspective:
+From the servers perspective:
- Receives StartupPacket
- |
- |
- (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
- | No
- |
- | Yes
+ Receives StartupPacket
+ |
+ |
+ (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
+ | No
+ |
+ | Yes
+ |
+ |
+ (Server compiled with USE_SSL?) ------- Send 'N'
+ | No |
+ | |
+ | Yes Normal startup
+ |
+ |
+ Send 'S'
+ |
+ |
+ Establish SSL
+ |
+ |
+ Normal startup
+
+
+
+
+
+From the clients perspective (v6.6 client _with_ SSL):
+
+
+ Connect
|
|
- (Server compiled with USE_SSL?) ------- Send 'N'
- | No |
- | |
- | Yes Normal startup
+ Send packet with SSL_NEGOTIATE_CODE
|
|
- Send 'S'
+ Receive single char ------- 'S' -------- Establish SSL
+ | |
+ | '<else>' |
+ | Normal startup
|
|
- Establish SSL
+ Is it 'E' for error ------------------- Retry connection
+ | Yes without SSL
+ | No
|
+ Is it 'N' for normal ------------------- Normal startup
+ | Yes
|
- Normal startup
-
-
-
-
-
->From the clients perspective (v6.6 client _with_ SSL):
-
-
- Connect
- |
- |
- Send packet with SSL_NEGOTIATE_CODE
- |
- |
- Receive single char ------- 'S' -------- Establish SSL
- | |
- | '<else>' |
- | Normal startup
- |
- |
- Is it 'E' for error ------------------- Retry connection
- | Yes without SSL
- | No
- |
- Is it 'N' for normal ------------------- Normal startup
- | Yes
- |
- Fail with unknown
+ Fail with unknown
---------------------------------------------------------------------------
diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README
index 2ab4f3dbf3..5805b61a23 100644
--- a/src/backend/optimizer/README
+++ b/src/backend/optimizer/README
@@ -254,7 +254,9 @@ the boundary, unless the proposed join is a LEFT join that can associate
into the SpecialJoinInfo's RHS using identity 3.
The use of minimum Relid sets has some pitfalls; consider a query like
+
A leftjoin (B leftjoin (C innerjoin D) on (Pbcd)) on Pa
+
where Pa doesn't mention B/C/D at all. In this case a naive computation
would give the upper leftjoin's min LHS as {A} and min RHS as {C,D} (since
we know that the innerjoin can't associate out of the leftjoin's RHS, and
@@ -262,7 +264,9 @@ enforce that by including its relids in the leftjoin's min RHS). And the
lower leftjoin has min LHS of {B} and min RHS of {C,D}. Given such
information, join_is_legal would think it's okay to associate the upper
join into the lower join's RHS, transforming the query to
+
B leftjoin (A leftjoin (C innerjoin D) on Pa) on (Pbcd)
+
which yields totally wrong answers. We prevent that by forcing the min RHS
for the upper join to include B. This is perhaps overly restrictive, but
such cases don't arise often so it's not clear that it's worth developing a
@@ -359,10 +363,12 @@ side of the full join a Var came from; but that information can be found
elsewhere at need.)
Notionally, a Var having nonempty varnullingrels can be thought of as
+
CASE WHEN any-of-these-outer-joins-produced-a-null-extended-row
THEN NULL
ELSE the-scan-level-value-of-the-column
END
+
It's only notional, because no such calculation is ever done explicitly.
In a finished plan, Vars occurring in scan-level plan nodes represent
the actual table column values, but upper-level Vars are always
@@ -375,14 +381,20 @@ otherwise be essential information for FULL JOIN cases.
Outer join identity 3 (discussed above) complicates this picture
a bit. In the form
+
A leftjoin (B leftjoin C on (Pbc)) on (Pab)
+
all of the Vars in clauses Pbc and Pab will have empty varnullingrels,
but if we start with
+
(A leftjoin B on (Pab)) leftjoin C on (Pbc)
+
then the parser will have marked Pbc's B Vars with the A/B join's
RT index, making this form artificially different from the first.
For discussion's sake, let's denote this marking with a star:
+
(A leftjoin B on (Pab)) leftjoin C on (Pb*c)
+
To cope with this, once we have detected that commuting these joins
is legal, we generate both the Pbc and Pb*c forms of that ON clause,
by either removing or adding the first join's RT index in the B Vars
@@ -553,114 +565,114 @@ Optimizer Functions
The primary entry point is planner().
-planner()
-set up for recursive handling of subqueries
--subquery_planner()
- pull up sublinks and subqueries from rangetable, if possible
- canonicalize qual
- Attempt to simplify WHERE clause to the most useful form; this includes
- flattening nested AND/ORs and detecting clauses that are duplicated in
- different branches of an OR.
- simplify constant expressions
- process sublinks
- convert Vars of outer query levels into Params
---grouping_planner()
- preprocess target list for non-SELECT queries
- handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
- ORDER BY, DISTINCT, LIMIT
----query_planner()
- make list of base relations used in query
- split up the qual into restrictions (a=1) and joins (b=c)
- find qual clauses that enable merge and hash joins
-----make_one_rel()
- set_base_rel_pathlists()
- find seqscan and all index paths for each base relation
- find selectivity of columns used in joins
- make_rel_from_joinlist()
- hand off join subproblems to a plugin, GEQO, or standard_join_search()
-------standard_join_search()
- call join_search_one_level() for each level of join tree needed
- join_search_one_level():
- For each joinrel of the prior level, do make_rels_by_clause_joins()
- if it has join clauses, or make_rels_by_clauseless_joins() if not.
- Also generate "bushy plan" joins between joinrels of lower levels.
- Back at standard_join_search(), generate gather paths if needed for
- each newly constructed joinrel, then apply set_cheapest() to extract
- the cheapest path for it.
- Loop back if this wasn't the top join level.
- Back at grouping_planner:
- do grouping (GROUP BY) and aggregation
- do window functions
- make unique (DISTINCT)
- do sorting (ORDER BY)
- do limit (LIMIT/OFFSET)
-Back at planner():
-convert finished Path tree into a Plan tree
-do final cleanup after planning
+ planner()
+ set up for recursive handling of subqueries
+ -subquery_planner()
+ pull up sublinks and subqueries from rangetable, if possible
+ canonicalize qual
+ Attempt to simplify WHERE clause to the most useful form; this includes
+ flattening nested AND/ORs and detecting clauses that are duplicated in
+ different branches of an OR.
+ simplify constant expressions
+ process sublinks
+ convert Vars of outer query levels into Params
+ --grouping_planner()
+ preprocess target list for non-SELECT queries
+ handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
+ ORDER BY, DISTINCT, LIMIT
+ ---query_planner()
+ make list of base relations used in query
+ split up the qual into restrictions (a=1) and joins (b=c)
+ find qual clauses that enable merge and hash joins
+ ----make_one_rel()
+ set_base_rel_pathlists()
+ find seqscan and all index paths for each base relation
+ find selectivity of columns used in joins
+ make_rel_from_joinlist()
+ hand off join subproblems to a plugin, GEQO, or standard_join_search()
+ ------standard_join_search()
+ call join_search_one_level() for each level of join tree needed
+ join_search_one_level():
+ For each joinrel of the prior level, do make_rels_by_clause_joins()
+ if it has join clauses, or make_rels_by_clauseless_joins() if not.
+ Also generate "bushy plan" joins between joinrels of lower levels.
+ Back at standard_join_search(), generate gather paths if needed for
+ each newly constructed joinrel, then apply set_cheapest() to extract
+ the cheapest path for it.
+ Loop back if this wasn't the top join level.
+ Back at grouping_planner:
+ do grouping (GROUP BY) and aggregation
+ do window functions
+ make unique (DISTINCT)
+ do sorting (ORDER BY)
+ do limit (LIMIT/OFFSET)
+ Back at planner():
+ convert finished Path tree into a Plan tree
+ do final cleanup after planning
Optimizer Data Structures
-------------------------
-PlannerGlobal - global information for a single planner invocation
-
-PlannerInfo - information for planning a particular Query (we make
- a separate PlannerInfo node for each sub-Query)
-
-RelOptInfo - a relation or joined relations
-
- RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
- (note the same structure is used for restriction and
- join clauses)
-
- Path - every way to generate a RelOptInfo(sequential,index,joins)
- A plain Path node can represent several simple plans, per its pathtype:
- T_SeqScan - sequential scan
- T_SampleScan - tablesample scan
- T_FunctionScan - function-in-FROM scan
- T_TableFuncScan - table function scan
- T_ValuesScan - VALUES scan
- T_CteScan - CTE (WITH) scan
- T_NamedTuplestoreScan - ENR scan
- T_WorkTableScan - scan worktable of a recursive CTE
- T_Result - childless Result plan node (used for FROM-less SELECT)
- IndexPath - index scan
- BitmapHeapPath - top of a bitmapped index scan
- TidPath - scan by CTID
- TidRangePath - scan a contiguous range of CTIDs
- SubqueryScanPath - scan a subquery-in-FROM
- ForeignPath - scan a foreign table, foreign join or foreign upper-relation
- CustomPath - for custom scan providers
- AppendPath - append multiple subpaths together
- MergeAppendPath - merge multiple subpaths, preserving their common sort order
- GroupResultPath - childless Result plan node (used for degenerate grouping)
- MaterialPath - a Material plan node
- MemoizePath - a Memoize plan node for caching tuples from sub-paths
- UniquePath - remove duplicate rows (either by hashing or sorting)
- GatherPath - collect the results of parallel workers
- GatherMergePath - collect parallel results, preserving their common sort order
- ProjectionPath - a Result plan node with child (used for projection)
- ProjectSetPath - a ProjectSet plan node applied to some sub-path
- SortPath - a Sort plan node applied to some sub-path
- IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
- GroupPath - a Group plan node applied to some sub-path
- UpperUniquePath - a Unique plan node applied to some sub-path
- AggPath - an Agg plan node applied to some sub-path
- GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
- MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
- WindowAggPath - a WindowAgg plan node applied to some sub-path
- SetOpPath - a SetOp plan node applied to some sub-path
- RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
- LockRowsPath - a LockRows plan node applied to some sub-path
- ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
- LimitPath - a Limit plan node applied to some sub-path
- NestPath - nested-loop joins
- MergePath - merge joins
- HashPath - hash joins
-
- EquivalenceClass - a data structure representing a set of values known equal
-
- PathKey - a data structure representing the sort ordering of a path
+ PlannerGlobal - global information for a single planner invocation
+
+ PlannerInfo - information for planning a particular Query (we make
+ a separate PlannerInfo node for each sub-Query)
+
+ RelOptInfo - a relation or joined relations
+
+ RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
+ (note the same structure is used for restriction and
+ join clauses)
+
+ Path - every way to generate a RelOptInfo(sequential,index,joins)
+ A plain Path node can represent several simple plans, per its pathtype:
+ T_SeqScan - sequential scan
+ T_SampleScan - tablesample scan
+ T_FunctionScan - function-in-FROM scan
+ T_TableFuncScan - table function scan
+ T_ValuesScan - VALUES scan
+ T_CteScan - CTE (WITH) scan
+ T_NamedTuplestoreScan - ENR scan
+ T_WorkTableScan - scan worktable of a recursive CTE
+ T_Result - childless Result plan node (used for FROM-less SELECT)
+ IndexPath - index scan
+ BitmapHeapPath - top of a bitmapped index scan
+ TidPath - scan by CTID
+ TidRangePath - scan a contiguous range of CTIDs
+ SubqueryScanPath - scan a subquery-in-FROM
+ ForeignPath - scan a foreign table, foreign join or foreign upper-relation
+ CustomPath - for custom scan providers
+ AppendPath - append multiple subpaths together
+ MergeAppendPath - merge multiple subpaths, preserving their common sort order
+ GroupResultPath - childless Result plan node (used for degenerate grouping)
+ MaterialPath - a Material plan node
+ MemoizePath - a Memoize plan node for caching tuples from sub-paths
+ UniquePath - remove duplicate rows (either by hashing or sorting)
+ GatherPath - collect the results of parallel workers
+ GatherMergePath - collect parallel results, preserving their common sort order
+ ProjectionPath - a Result plan node with child (used for projection)
+ ProjectSetPath - a ProjectSet plan node applied to some sub-path
+ SortPath - a Sort plan node applied to some sub-path
+ IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
+ GroupPath - a Group plan node applied to some sub-path
+ UpperUniquePath - a Unique plan node applied to some sub-path
+ AggPath - an Agg plan node applied to some sub-path
+ GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
+ MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
+ WindowAggPath - a WindowAgg plan node applied to some sub-path
+ SetOpPath - a SetOp plan node applied to some sub-path
+ RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
+ LockRowsPath - a LockRows plan node applied to some sub-path
+ ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
+ LimitPath - a Limit plan node applied to some sub-path
+ NestPath - nested-loop joins
+ MergePath - merge joins
+ HashPath - hash joins
+
+ EquivalenceClass - a data structure representing a set of values known equal
+
+ PathKey - a data structure representing the sort ordering of a path
The optimizer spends a good deal of its time worrying about the ordering
of the tuples returned by a path. The reason this is useful is that by
@@ -909,10 +921,10 @@ of the tuples generated by a particular Path. A path's pathkeys field is a
list of PathKey nodes, where the n'th item represents the n'th sort key of
the result. Each PathKey contains these fields:
- * a reference to an EquivalenceClass
- * a btree opfamily OID (must match one of those in the EC)
- * a sort direction (ascending or descending)
- * a nulls-first-or-last flag
+* a reference to an EquivalenceClass
+* a btree opfamily OID (must match one of those in the EC)
+* a sort direction (ascending or descending)
+* a nulls-first-or-last flag
The EquivalenceClass represents the value being sorted on. Since the
various members of an EquivalenceClass are known equal according to the
@@ -997,9 +1009,11 @@ lists (sort orderings) do not mention the same EquivalenceClass more than
once. For example, in all these cases the second sort column is redundant,
because it cannot distinguish values that are the same according to the
first sort column:
+
SELECT ... ORDER BY x, x
SELECT ... ORDER BY x, x DESC
SELECT ... WHERE x = y ORDER BY x, y
+
Although a user probably wouldn't write "ORDER BY x,x" directly, such
redundancies are more probable once equivalence classes have been
considered. Also, the system may generate redundant pathkey lists when
@@ -1350,14 +1364,14 @@ RelOptInfos are mostly dummy, but their pathlist lists hold all the Paths
considered useful for each step. Currently, we may create these types of
additional RelOptInfos during upper-level planning:
-UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
-UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
-UPPERREL_GROUP_AGG result of grouping/aggregation, if any
-UPPERREL_WINDOW result of window functions, if any
-UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
-UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
-UPPERREL_ORDERED result of ORDER BY, if any
-UPPERREL_FINAL result of any remaining top-level actions
+ UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
+ UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
+ UPPERREL_GROUP_AGG result of grouping/aggregation, if any
+ UPPERREL_WINDOW result of window functions, if any
+ UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
+ UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
+ UPPERREL_ORDERED result of ORDER BY, if any
+ UPPERREL_FINAL result of any remaining top-level actions
UPPERREL_FINAL is used to represent any final processing steps, currently
LockRows (SELECT FOR UPDATE), LIMIT/OFFSET, and ModifyTable. There is no
diff --git a/src/backend/optimizer/plan/README b/src/backend/optimizer/plan/README
index 013c0f9ea2..93dd422dc1 100644
--- a/src/backend/optimizer/plan/README
+++ b/src/backend/optimizer/plan/README
@@ -6,32 +6,32 @@ Subselects
Vadim B. Mikheev
-From [email protected] Fri Feb 13 09:01:19 1998
-Received: from renoir.op.net ([email protected] [209.152.193.4])
- by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
- for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
-Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
-Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
-Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
-Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
-Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
-Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
- by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
- for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
- (envelope-from [email protected])
-Message-ID: <[email protected]>
-Date: Fri, 13 Feb 1998 20:50:50 +0700
-From: "Vadim B. Mikheev" <[email protected]>
-Organization: ITTS (Krasnoyarsk)
-X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
-MIME-Version: 1.0
-To: PostgreSQL Developers List <[email protected]>
-Subject: [HACKERS] Subselects are in CVS...
-Content-Type: text/plain; charset=us-ascii
-Content-Transfer-Encoding: 7bit
-Sender: [email protected]
-Precedence: bulk
-Status: OR
+ From [email protected] Fri Feb 13 09:01:19 1998
+ Received: from renoir.op.net ([email protected] [209.152.193.4])
+ by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
+ for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
+ Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
+ Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
+ Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
+ Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
+ Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
+ Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
+ by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
+ for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
+ (envelope-from [email protected])
+ Message-ID: <[email protected]>
+ Date: Fri, 13 Feb 1998 20:50:50 +0700
+ From: "Vadim B. Mikheev" <[email protected]>
+ Organization: ITTS (Krasnoyarsk)
+ X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
+ MIME-Version: 1.0
+ To: PostgreSQL Developers List <[email protected]>
+ Subject: [HACKERS] Subselects are in CVS...
+ Content-Type: text/plain; charset=us-ascii
+ Content-Transfer-Encoding: 7bit
+ Sender: [email protected]
+ Precedence: bulk
+ Status: OR
This is some implementation notes and opened issues...
@@ -95,17 +95,17 @@ vac=> explain select * from tmp where x >= (select max(x2) from test2
where y2 = y and exists (select * from tempx where tx = x));
NOTICE: QUERY PLAN:
-Seq Scan on tmp (cost=40.03 size=101 width=8)
- SubPlan
- ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
- -> Aggregate (cost=2.05 size=0 width=0)
- InitPlan
- ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
- -> Seq Scan on tempx (cost=4.33 size=1 width=4)
- -> Result (cost=2.05 size=0 width=0)
- ^^^^^^ EXISTS subsubquery was transformed into Param
- and so we have Result node here
- -> Index Scan on test2 (cost=2.05 size=1 width=4)
+ Seq Scan on tmp (cost=40.03 size=101 width=8)
+ SubPlan
+ ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
+ -> Aggregate (cost=2.05 size=0 width=0)
+ InitPlan
+ ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
+ -> Seq Scan on tempx (cost=4.33 size=1 width=4)
+ -> Result (cost=2.05 size=0 width=0)
+ ^^^^^^ EXISTS subsubquery was transformed into Param
+ and so we have Result node here
+ -> Index Scan on test2 (cost=2.05 size=1 width=4)
Opened issues.
@@ -131,28 +131,28 @@ Results of some test. TMP is table with x,y (int4-s), x in 0-9,
y = 100 - x, 1000 tuples (10 duplicates of each tuple). TEST2 is table
with x2, y2 (int4-s), x2 in 1-99, y2 = 100 -x2, 10000 tuples (100 dups).
- Trying
+Trying
-select * from tmp where x >= (select max(x2) from test2 where y2 = y);
+ select * from tmp where x >= (select max(x2) from test2 where y2 = y);
- and
+and
-begin;
-select y as ty, max(x2) as mx into table tsub from test2, tmp
-where y2 = y group by ty;
-vacuum tsub;
-select x, y from tmp, tsub where x >= mx and y = ty;
-drop table tsub;
-end;
+ begin;
+ select y as ty, max(x2) as mx into table tsub from test2, tmp
+ where y2 = y group by ty;
+ vacuum tsub;
+ select x, y from tmp, tsub where x >= mx and y = ty;
+ drop table tsub;
+ end;
- Without index on test2(y2):
+Without index on test2(y2):
-SubSelect -> 320 sec
-Using temp table -> 32 sec
+ SubSelect -> 320 sec
+ Using temp table -> 32 sec
- Having index
+Having index
-SubSelect -> 17 sec (2M of memory)
-Using temp table -> 32 sec (12M of memory: -S 8192)
+ SubSelect -> 17 sec (2M of memory)
+ Using temp table -> 32 sec (12M of memory: -S 8192)
Vadim
diff --git a/src/backend/parser/README b/src/backend/parser/README
index e0c986a41e..e6016fa430 100644
--- a/src/backend/parser/README
+++ b/src/backend/parser/README
@@ -7,27 +7,27 @@ This directory does more than tokenize and parse SQL queries. It also
creates Query structures for the various complex queries that are passed
to the optimizer and then executor.
-parser.c things start here
-scan.l break query into tokens
-scansup.c handle escapes in input strings
-gram.y parse the tokens and produce a "raw" parse tree
-analyze.c top level of parse analysis for optimizable queries
-parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
-parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
-parse_coerce.c handle coercing expressions to different data types
-parse_collate.c assign collation information in completed expressions
-parse_cte.c handle Common Table Expressions (WITH clauses)
-parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
-parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
-parse_func.c handle functions, table.column and column identifiers
-parse_merge.c handle MERGE
-parse_node.c create nodes for various structures
-parse_oper.c handle operators in expressions
-parse_param.c handle Params (for the cases used in the core backend)
-parse_relation.c support routines for tables and column handling
-parse_target.c handle the result list of the query
-parse_type.c support routines for data type handling
-parse_utilcmd.c parse analysis for utility commands (done at execution time)
+ parser.c things start here
+ scan.l break query into tokens
+ scansup.c handle escapes in input strings
+ gram.y parse the tokens and produce a "raw" parse tree
+ analyze.c top level of parse analysis for optimizable queries
+ parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
+ parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
+ parse_coerce.c handle coercing expressions to different data types
+ parse_collate.c assign collation information in completed expressions
+ parse_cte.c handle Common Table Expressions (WITH clauses)
+ parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
+ parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
+ parse_func.c handle functions, table.column and column identifiers
+ parse_merge.c handle MERGE
+ parse_node.c create nodes for various structures
+ parse_oper.c handle operators in expressions
+ parse_param.c handle Params (for the cases used in the core backend)
+ parse_relation.c support routines for tables and column handling
+ parse_target.c handle the result list of the query
+ parse_type.c support routines for data type handling
+ parse_utilcmd.c parse analysis for utility commands (done at execution time)
See also src/common/keywords.c, which contains the table of standard
keywords and the keyword lookup function. We separated that out because
diff --git a/src/backend/regex/README b/src/backend/regex/README
index 930d8ced0d..a0d45589b6 100644
--- a/src/backend/regex/README
+++ b/src/backend/regex/README
@@ -9,11 +9,13 @@ General source-file layout
There are six separately-compilable source files, five of which expose
exactly one exported function apiece:
- regcomp.c: pg_regcomp
- regexec.c: pg_regexec
- regerror.c: pg_regerror
- regfree.c: pg_regfree
- regprefix.c: pg_regprefix
+
+ regcomp.c pg_regcomp
+ regexec.c pg_regexec
+ regerror.c pg_regerror
+ regfree.c pg_regfree
+ regprefix.c pg_regprefix
+
(The pg_ prefixes were added by the Postgres project to distinguish this
library version from any similar one that might be present on a particular
system. They'd need to be removed or replaced in any standalone version
@@ -38,19 +40,19 @@ structs.)
What's where in src/backend/regex/:
-regcomp.c Top-level regex compilation code
-regc_color.c Color map management
-regc_cvec.c Character vector (cvec) management
-regc_lex.c Lexer
-regc_nfa.c NFA handling
-regc_locale.c Application-specific locale code from Tcl project
-regc_pg_locale.c Postgres-added application-specific locale code
-regexec.c Top-level regex execution code
-rege_dfa.c DFA creation and execution
-regerror.c pg_regerror: generate text for a regex error code
-regfree.c pg_regfree: API to free a no-longer-needed regex_t
-regexport.c Functions for extracting info from a regex_t
-regprefix.c Code for extracting a common prefix from a regex_t
+ regcomp.c Top-level regex compilation code
+ regc_color.c Color map management
+ regc_cvec.c Character vector (cvec) management
+ regc_lex.c Lexer
+ regc_nfa.c NFA handling
+ regc_locale.c Application-specific locale code from Tcl project
+ regc_pg_locale.c Postgres-added application-specific locale code
+ regexec.c Top-level regex execution code
+ rege_dfa.c DFA creation and execution
+ regerror.c pg_regerror: generate text for a regex error code
+ regfree.c pg_regfree: API to free a no-longer-needed regex_t
+ regexport.c Functions for extracting info from a regex_t
+ regprefix.c Code for extracting a common prefix from a regex_t
The locale-specific code is concerned primarily with case-folding and with
expanding locale-specific character classes, such as [[:alnum:]]. It
@@ -58,11 +60,11 @@ really needs refactoring if this is ever to become a standalone library.
The header files for the library are in src/include/regex/:
-regcustom.h Customizes library for particular application
-regerrs.h Error message list
-regex.h Exported API
-regexport.h Exported API for regexport.c
-regguts.h Internals declarations
+ regcustom.h Customizes library for particular application
+ regerrs.h Error message list
+ regex.h Exported API
+ regexport.h Exported API for regexport.c
+ regguts.h Internals declarations
DFAs, NFAs, and all that
@@ -261,15 +263,19 @@ an additional arc labeled 2 wherever there is an arc labeled 3; this
action ensures that characters of color 2 (i.e., "x") will still be
considered as allowing any transitions they did before. We are now done
parsing the regex, and we have these final color assignments:
+
color 1: "a"
color 2: "x"
color 3: other letters
color 4: digits
+
and the NFA has these arcs:
+
states 1 -> 2 on color 1 (hence, "a" only)
states 2 -> 3 on color 4 (digits)
states 3 -> 4 on colors 1, 3, 4, and 2 (covering all \w characters)
states 4 -> 5 on color 2 ("x" only)
+
which can be seen to be a correct representation of the regex.
There is one more complexity, which is how to handle ".", that is a
diff --git a/src/backend/snowball/README b/src/backend/snowball/README
index 675baff5c9..ab4e4044d3 100644
--- a/src/backend/snowball/README
+++ b/src/backend/snowball/README
@@ -10,12 +10,12 @@ which is released by them under a BSD-style license.
The Snowball project does not often make formal releases; it's best
to pull from their git repository
-git clone https://github.com/snowballstem/snowball.git
+ git clone https://github.com/snowballstem/snowball.git
and then building the derived files is as simple as
-cd snowball
-make
+ cd snowball
+ make
At least on Linux, no platform-specific adjustment is needed.
@@ -39,10 +39,10 @@ To update the PostgreSQL sources from a new Snowball version:
1. Copy the *.c files in snowball/src_c/ to src/backend/snowball/libstemmer
with replacement of "../runtime/header.h" by "header.h", for example
-for f in .../snowball/src_c/*.c
-do
- sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
-done
+ for f in .../snowball/src_c/*.c
+ do
+ sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
+ done
Do not copy stemmers that are listed in libstemmer/modules.txt as
nonstandard, such as "german2" or "lovins".
diff --git a/src/backend/storage/freespace/README b/src/backend/storage/freespace/README
index dc2a63a137..6d8882c3da 100644
--- a/src/backend/storage/freespace/README
+++ b/src/backend/storage/freespace/README
@@ -33,9 +33,9 @@ node stores the max amount of free space on any of its children.
For example:
- 4
- 4 2
-3 4 0 2 <- This level represents heap pages
+ 4
+ 4 2
+ 3 4 0 2 <- This level represents heap pages
We need two basic operations: search and update.
@@ -67,10 +67,10 @@ header takes some space on a page, the binary tree isn't perfect. That is,
a few right-most leaf nodes are missing, and there are some useless non-leaf
nodes at the right. So the tree looks something like this:
- 0
- 1 2
- 3 4 5 6
-7 8 9 A B
+ 0
+ 1 2
+ 3 4 5 6
+ 7 8 9 A B
where the numbers denote each node's position in the array. Note that the
tree is guaranteed complete above the leaf level; only some leaf nodes are
@@ -100,27 +100,27 @@ For example, assuming each FSM page can hold information about 4 pages (in
reality, it holds (BLCKSZ - headers) / 2, or ~4000 with default BLCKSZ),
we get a disk layout like this:
- 0 <-- page 0 at level 2 (root page)
- 0 <-- page 0 at level 1
- 0 <-- page 0 at level 0
- 1 <-- page 1 at level 0
- 2 <-- ...
- 3
- 1 <-- page 1 at level 1
- 4
- 5
- 6
- 7
- 2
- 8
- 9
- 10
- 11
- 3
- 12
- 13
- 14
- 15
+ 0 <-- page 0 at level 2 (root page)
+ 0 <-- page 0 at level 1
+ 0 <-- page 0 at level 0
+ 1 <-- page 1 at level 0
+ 2 <-- ...
+ 3
+ 1 <-- page 1 at level 1
+ 4
+ 5
+ 6
+ 7
+ 2
+ 8
+ 9
+ 10
+ 11
+ 3
+ 12
+ 13
+ 14
+ 15
where the numbers are page numbers *at that level*, starting from 0.
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI
index 50d2ecca9d..48d6e025d8 100644
--- a/src/backend/storage/lmgr/README-SSI
+++ b/src/backend/storage/lmgr/README-SSI
@@ -199,55 +199,55 @@ The PostgreSQL implementation uses two additional optimizations:
PostgreSQL Implementation
-------------------------
- * Since this technique is based on Snapshot Isolation (SI), those
-areas in PostgreSQL which don't use SI can't be brought under SSI.
-This includes system tables, temporary tables, sequences, hint bit
-rewrites, etc. SSI can not eliminate existing anomalies in these
-areas.
-
- * Any transaction which is run at a transaction isolation level
-other than SERIALIZABLE will not be affected by SSI. If you want to
-enforce business rules through SSI, all transactions should be run at
-the SERIALIZABLE transaction isolation level, and that should
-probably be set as the default.
-
- * If all transactions are run at the SERIALIZABLE transaction
-isolation level, business rules can be enforced in triggers or
-application code without ever having a need to acquire an explicit
-lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
-
- * Those who want to continue to use snapshot isolation without
-the additional protections of SSI (and the associated costs of
-enforcing those protections), can use the REPEATABLE READ transaction
-isolation level. This level retains its legacy behavior, which
-is identical to the old SERIALIZABLE implementation and fully
-consistent with the standard's requirements for the REPEATABLE READ
-transaction isolation level.
-
- * Performance under this SSI implementation will be significantly
-improved if transactions which don't modify permanent tables are
-declared to be READ ONLY before they begin reading data.
-
- * Performance under SSI will tend to degrade more rapidly with a
-large number of active database transactions than under less strict
-isolation levels. Limiting the number of active transactions through
-use of a connection pool or similar techniques may be necessary to
-maintain good performance.
-
- * Any transaction which must be rolled back to prevent
-serialization anomalies will fail with SQLSTATE 40001, which has a
-standard meaning of "serialization failure".
-
- * This SSI implementation makes an effort to choose the
-transaction to be canceled such that an immediate retry of the
-transaction will not fail due to conflicts with exactly the same
-transactions. Pursuant to this goal, no transaction is canceled
-until one of the other transactions in the set of conflicts which
-could generate an anomaly has successfully committed. This is
-conceptually similar to how write conflicts are handled. To fully
-implement this guarantee there needs to be a way to roll back the
-active transaction for another process with a serialization failure
-SQLSTATE, even if it is "idle in transaction".
+* Since this technique is based on Snapshot Isolation (SI), those
+ areas in PostgreSQL which don't use SI can't be brought under SSI.
+ This includes system tables, temporary tables, sequences, hint bit
+ rewrites, etc. SSI can not eliminate existing anomalies in these
+ areas.
+
+* Any transaction which is run at a transaction isolation level
+ other than SERIALIZABLE will not be affected by SSI. If you want to
+ enforce business rules through SSI, all transactions should be run at
+ the SERIALIZABLE transaction isolation level, and that should
+ probably be set as the default.
+
+* If all transactions are run at the SERIALIZABLE transaction
+ isolation level, business rules can be enforced in triggers or
+ application code without ever having a need to acquire an explicit
+ lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
+
+* Those who want to continue to use snapshot isolation without
+ the additional protections of SSI (and the associated costs of
+ enforcing those protections), can use the REPEATABLE READ transaction
+ isolation level. This level retains its legacy behavior, which
+ is identical to the old SERIALIZABLE implementation and fully
+ consistent with the standard's requirements for the REPEATABLE READ
+ transaction isolation level.
+
+* Performance under this SSI implementation will be significantly
+ improved if transactions which don't modify permanent tables are
+ declared to be READ ONLY before they begin reading data.
+
+* Performance under SSI will tend to degrade more rapidly with a
+ large number of active database transactions than under less strict
+ isolation levels. Limiting the number of active transactions through
+ use of a connection pool or similar techniques may be necessary to
+ maintain good performance.
+
+* Any transaction which must be rolled back to prevent
+ serialization anomalies will fail with SQLSTATE 40001, which has a
+ standard meaning of "serialization failure".
+
+* This SSI implementation makes an effort to choose the
+ transaction to be canceled such that an immediate retry of the
+ transaction will not fail due to conflicts with exactly the same
+ transactions. Pursuant to this goal, no transaction is canceled
+ until one of the other transactions in the set of conflicts which
+ could generate an anomaly has successfully committed. This is
+ conceptually similar to how write conflicts are handled. To fully
+ implement this guarantee there needs to be a way to roll back the
+ active transaction for another process with a serialization failure
+ SQLSTATE, even if it is "idle in transaction".
Predicate Locking
@@ -303,23 +303,23 @@ Heap locking
Predicate locks will be acquired for the heap based on the following:
- * For a table scan, the entire relation will be locked.
+* For a table scan, the entire relation will be locked.
- * Each tuple read which is visible to the reading transaction
-will be locked, whether or not it meets selection criteria; except
-that there is no need to acquire an SIREAD lock on a tuple when the
-transaction already holds a write lock on any tuple representing the
-row, since a rw-conflict would also create a ww-dependency which
-has more aggressive enforcement and thus will prevent any anomaly.
+* Each tuple read which is visible to the reading transaction
+ will be locked, whether or not it meets selection criteria; except
+ that there is no need to acquire an SIREAD lock on a tuple when the
+ transaction already holds a write lock on any tuple representing the
+ row, since a rw-conflict would also create a ww-dependency which
+ has more aggressive enforcement and thus will prevent any anomaly.
- * Modifying a heap tuple creates a rw-conflict with any transaction
-that holds a SIREAD lock on that tuple, or on the page or relation
-that contains it.
+* Modifying a heap tuple creates a rw-conflict with any transaction
+ that holds a SIREAD lock on that tuple, or on the page or relation
+ that contains it.
- * Inserting a new tuple creates a rw-conflict with any transaction
-holding a SIREAD lock on the entire relation. It doesn't conflict with
-page-level locks, because page-level locks are only used to aggregate
-tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
+* Inserting a new tuple creates a rw-conflict with any transaction
+ holding a SIREAD lock on the entire relation. It doesn't conflict with
+ page-level locks, because page-level locks are only used to aggregate
+ tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
Index AM implementations
@@ -346,60 +346,60 @@ false positives, they should be minimized for performance reasons.
Several optimizations are possible, though not all are implemented yet:
- * An index scan which is just finding the right position for an
-index insertion or deletion need not acquire a predicate lock.
+* An index scan which is just finding the right position for an
+ index insertion or deletion need not acquire a predicate lock.
- * An index scan which is comparing for equality on the entire key
-for a unique index need not acquire a predicate lock as long as a key
-is found corresponding to a visible tuple which has not been modified
-by another transaction -- there are no "between or around" gaps to
-cover.
+* An index scan which is comparing for equality on the entire key
+ for a unique index need not acquire a predicate lock as long as a key
+ is found corresponding to a visible tuple which has not been modified
+ by another transaction -- there are no "between or around" gaps to
+ cover.
- * As long as built-in foreign key enforcement continues to use
-its current "special tricks" to deal with MVCC issues, predicate
-locks should not be needed for scans done by enforcement code.
+* As long as built-in foreign key enforcement continues to use
+ its current "special tricks" to deal with MVCC issues, predicate
+ locks should not be needed for scans done by enforcement code.
- * If a search determines that no rows can be found regardless of
-index contents because the search conditions are contradictory (e.g.,
-x = 1 AND x = 2), then no predicate lock is needed.
+* If a search determines that no rows can be found regardless of
+ index contents because the search conditions are contradictory (e.g.,
+ x = 1 AND x = 2), then no predicate lock is needed.
Other index AM implementation considerations:
- * For an index AM that doesn't have support for predicate locking,
-we just acquire a predicate lock on the whole index for any search.
-
- * B-tree index searches acquire predicate locks only on the
-index *leaf* pages needed to lock the appropriate index range. If,
-however, a search discovers that no root page has yet been created, a
-predicate lock on the index relation is required.
-
- * Like a B-tree, GIN searches acquire predicate locks only on the
-leaf pages of entry tree. When performing an equality scan, and an
-entry has a posting tree, the posting tree root is locked instead, to
-lock only that key value. However, fastupdate=on postpones the
-insertion of tuples into index structure by temporarily storing them
-into pending list. That makes us unable to detect r-w conflicts using
-page-level locks. To cope with that, insertions to the pending list
-conflict with all scans.
-
- * GiST searches can determine that there are no matches at any
-level of the index, so we acquire predicate lock at each index
-level during a GiST search. An index insert at the leaf level can
-then be trusted to ripple up to all levels and locations where
-conflicting predicate locks may exist. In case there is a page split,
-we need to copy predicate lock from the original page to all the new
-pages.
-
- * Hash index searches acquire predicate locks on the primary
-page of a bucket. It acquires a lock on both the old and new buckets
-for scans that happen concurrently with page splits. During a bucket
-split, a predicate lock is copied from the primary page of an old
-bucket to the primary page of a new bucket.
-
- * The effects of page splits, overflows, consolidations, and
-removals must be carefully reviewed to ensure that predicate locks
-aren't "lost" during those operations, or kept with pages which could
-get re-used for different parts of the index.
+* For an index AM that doesn't have support for predicate locking,
+ we just acquire a predicate lock on the whole index for any search.
+
+* B-tree index searches acquire predicate locks only on the
+ index *leaf* pages needed to lock the appropriate index range. If,
+ however, a search discovers that no root page has yet been created, a
+ predicate lock on the index relation is required.
+
+* Like a B-tree, GIN searches acquire predicate locks only on the
+ leaf pages of entry tree. When performing an equality scan, and an
+ entry has a posting tree, the posting tree root is locked instead, to
+ lock only that key value. However, fastupdate=on postpones the
+ insertion of tuples into index structure by temporarily storing them
+ into pending list. That makes us unable to detect r-w conflicts using
+ page-level locks. To cope with that, insertions to the pending list
+ conflict with all scans.
+
+* GiST searches can determine that there are no matches at any
+ level of the index, so we acquire predicate lock at each index
+ level during a GiST search. An index insert at the leaf level can
+ then be trusted to ripple up to all levels and locations where
+ conflicting predicate locks may exist. In case there is a page split,
+ we need to copy predicate lock from the original page to all the new
+ pages.
+
+* Hash index searches acquire predicate locks on the primary
+ page of a bucket. It acquires a lock on both the old and new buckets
+ for scans that happen concurrently with page splits. During a bucket
+ split, a predicate lock is copied from the primary page of an old
+ bucket to the primary page of a new bucket.
+
+* The effects of page splits, overflows, consolidations, and
+ removals must be carefully reviewed to ensure that predicate locks
+ aren't "lost" during those operations, or kept with pages which could
+ get re-used for different parts of the index.
Innovations
@@ -409,183 +409,183 @@ The PostgreSQL implementation of Serializable Snapshot Isolation
differs from what is described in the cited papers for several
reasons:
- 1. PostgreSQL didn't have any existing predicate locking. It had
+1. PostgreSQL didn't have any existing predicate locking. It had
to be added from scratch.
- 2. The existing in-memory lock structures were not suitable for
+2. The existing in-memory lock structures were not suitable for
tracking SIREAD locks.
- * In PostgreSQL, tuple level locks are not held in RAM for
+- In PostgreSQL, tuple level locks are not held in RAM for
any length of time; lock information is written to the tuples
involved in the transactions.
- * In PostgreSQL, existing lock structures have pointers to
+- In PostgreSQL, existing lock structures have pointers to
memory which is related to a session. SIREAD locks need to persist
past the end of the originating transaction and even the session
which ran it.
- * PostgreSQL needs to be able to tolerate a large number of
+- PostgreSQL needs to be able to tolerate a large number of
transactions executing while one long-running transaction stays open
-- the in-RAM techniques discussed in the papers wouldn't support
that.
- 3. Unlike the database products used for the prototypes described
+3. Unlike the database products used for the prototypes described
in the papers, PostgreSQL didn't already have a true serializable
isolation level distinct from snapshot isolation.
- 4. PostgreSQL supports subtransactions -- an issue not mentioned
+4. PostgreSQL supports subtransactions -- an issue not mentioned
in the papers.
- 5. PostgreSQL doesn't assign a transaction number to a database
+5. PostgreSQL doesn't assign a transaction number to a database
transaction until and unless necessary (normally, when the transaction
attempts to modify data).
- 6. PostgreSQL has pluggable data types with user-definable
+6. PostgreSQL has pluggable data types with user-definable
operators, as well as pluggable index types, not all of which are
based around data types which support ordering.
- 7. Some possible optimizations became apparent during development
+7. Some possible optimizations became apparent during development
and testing.
Differences from the implementation described in the papers are
listed below.
- * New structures needed to be created in shared memory to track
-the proper information for serializable transactions and their SIREAD
-locks.
-
- * Because PostgreSQL does not have the same concept of an "oldest
-transaction ID" for all serializable transactions as assumed in the
-Cahill thesis, we track the oldest snapshot xmin among serializable
-transactions, and a count of how many active transactions use that
-xmin. When the count hits zero we find the new oldest xmin and run a
-clean-up based on that.
-
- * Because reads in a subtransaction may cause that subtransaction
-to roll back, thereby affecting what is written by the top level
-transaction, predicate locks must survive a subtransaction rollback.
-As a consequence, all xid usage in SSI, including predicate locking,
-is based on the top level xid. When looking at an xid that comes
-from a tuple's xmin or xmax, for example, we always call
-SubTransGetTopmostTransaction() before doing much else with it.
-
- * PostgreSQL does not use "update in place" with a rollback log
-for its MVCC implementation. Where possible it uses "HOT" updates on
-the same page (if there is room and no indexed value is changed).
-For non-HOT updates the old tuple is expired in place and a new tuple
-is inserted at a new location. Because of this difference, a tuple
-lock in PostgreSQL doesn't automatically lock any other versions of a
-row. We don't try to copy or expand a tuple lock to any other
-versions of the row, based on the following proof that any additional
-serialization failures we would get from that would be false
-positives:
-
- o If transaction T1 reads a row version (thus acquiring a
-predicate lock on it) and a second transaction T2 updates that row
-version (thus creating a rw-conflict graph edge from T1 to T2), must a
-third transaction T3 which re-updates the new version of the row also
-have a rw-conflict in from T1 to prevent anomalies? In other words,
-does it matter whether we recognize the edge T1 -> T3?
-
- o If T1 has a conflict in, it certainly doesn't. Adding the
-edge T1 -> T3 would create a dangerous structure, but we already had
-one from the edge T1 -> T2, so we would have aborted something anyway.
-(T2 has already committed, else T3 could not have updated its output;
-but we would have aborted either T1 or T1's predecessor(s). Hence
-no cycle involving T1 and T3 can survive.)
-
- o Now let's consider the case where T1 doesn't have a
-rw-conflict in. If that's the case, for this edge T1 -> T3 to make a
-difference, T3 must have a rw-conflict out that induces a cycle in the
-dependency graph, i.e. a conflict out to some transaction preceding T1
-in the graph. (A conflict out to T1 itself would be problematic too,
-but that would mean T1 has a conflict in, the case we already
-eliminated.)
-
- o So now we're trying to figure out if there can be an
-rw-conflict edge T3 -> T0, where T0 is some transaction that precedes
-T1. For T0 to precede T1, there has to be some edge, or sequence of
-edges, from T0 to T1. At least the last edge has to be a wr-dependency
-or ww-dependency rather than a rw-conflict, because T1 doesn't have a
-rw-conflict in. And that gives us enough information about the order
-of transactions to see that T3 can't have a rw-conflict to T0:
- - T0 committed before T1 started (the wr/ww-dependency implies this)
- - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
- - T2 committed before T3 started (otherwise, T3 would get aborted
+* New structures needed to be created in shared memory to track
+ the proper information for serializable transactions and their SIREAD
+ locks.
+
+* Because PostgreSQL does not have the same concept of an "oldest
+ transaction ID" for all serializable transactions as assumed in the
+ Cahill thesis, we track the oldest snapshot xmin among serializable
+ transactions, and a count of how many active transactions use that
+ xmin. When the count hits zero we find the new oldest xmin and run a
+ clean-up based on that.
+
+* Because reads in a subtransaction may cause that subtransaction
+ to roll back, thereby affecting what is written by the top level
+ transaction, predicate locks must survive a subtransaction rollback.
+ As a consequence, all xid usage in SSI, including predicate locking,
+ is based on the top level xid. When looking at an xid that comes
+ from a tuple's xmin or xmax, for example, we always call
+ SubTransGetTopmostTransaction() before doing much else with it.
+
+* PostgreSQL does not use "update in place" with a rollback log
+ for its MVCC implementation. Where possible it uses "HOT" updates on
+ the same page (if there is room and no indexed value is changed).
+ For non-HOT updates the old tuple is expired in place and a new tuple
+ is inserted at a new location. Because of this difference, a tuple
+ lock in PostgreSQL doesn't automatically lock any other versions of a
+ row. We don't try to copy or expand a tuple lock to any other
+ versions of the row, based on the following proof that any additional
+ serialization failures we would get from that would be false
+ positives:
+
+ - If transaction T1 reads a row version (thus acquiring a
+ predicate lock on it) and a second transaction T2 updates that row
+ version (thus creating a rw-conflict graph edge from T1 to T2), must a
+ third transaction T3 which re-updates the new version of the row also
+ have a rw-conflict in from T1 to prevent anomalies? In other words,
+ does it matter whether we recognize the edge T1 -> T3?
+
+ - If T1 has a conflict in, it certainly doesn't. Adding the
+ edge T1 -> T3 would create a dangerous structure, but we already had
+ one from the edge T1 -> T2, so we would have aborted something anyway.
+ (T2 has already committed, else T3 could not have updated its output;
+ but we would have aborted either T1 or T1's predecessor(s). Hence
+ no cycle involving T1 and T3 can survive.)
+
+ - Now let's consider the case where T1 doesn't have a
+ rw-conflict in. If that's the case, for this edge T1 -> T3 to make a
+ difference, T3 must have a rw-conflict out that induces a cycle in the
+ dependency graph, i.e. a conflict out to some transaction preceding T1
+ in the graph. (A conflict out to T1 itself would be problematic too,
+ but that would mean T1 has a conflict in, the case we already
+ eliminated.)
+
+ - So now we're trying to figure out if there can be an
+ rw-conflict edge T3 -> T0, where T0 is some transaction that precedes
+ T1. For T0 to precede T1, there has to be some edge, or sequence of
+ edges, from T0 to T1. At least the last edge has to be a wr-dependency
+ or ww-dependency rather than a rw-conflict, because T1 doesn't have a
+ rw-conflict in. And that gives us enough information about the order
+ of transactions to see that T3 can't have a rw-conflict to T0:
+ - T0 committed before T1 started (the wr/ww-dependency implies this)
+ - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
+ - T2 committed before T3 started (otherwise, T3 would get aborted
because of an update conflict)
- o That means T0 committed before T3 started, and therefore
-there can't be a rw-conflict from T3 to T0.
+ - That means T0 committed before T3 started, and therefore
+ there can't be a rw-conflict from T3 to T0.
- o So in all cases, we don't need the T1 -> T3 edge to
-recognize cycles. Therefore it's not necessary for T1's SIREAD lock
-on the original tuple version to cover later versions as well.
+ - So in all cases, we don't need the T1 -> T3 edge to
+ recognize cycles. Therefore it's not necessary for T1's SIREAD lock
+ on the original tuple version to cover later versions as well.
- * Predicate locking in PostgreSQL starts at the tuple level
-when possible. Multiple fine-grained locks are promoted to a single
-coarser-granularity lock as needed to avoid resource exhaustion. The
-amount of memory used for these structures is configurable, to balance
-RAM usage against SIREAD lock granularity.
+* Predicate locking in PostgreSQL starts at the tuple level
+ when possible. Multiple fine-grained locks are promoted to a single
+ coarser-granularity lock as needed to avoid resource exhaustion. The
+ amount of memory used for these structures is configurable, to balance
+ RAM usage against SIREAD lock granularity.
- * Each backend keeps a process-local table of the locks it holds.
-To support granularity promotion decisions with low CPU and locking
-overhead, this table also includes the coarser covering locks and the
-number of finer-granularity locks they cover.
+* Each backend keeps a process-local table of the locks it holds.
+ To support granularity promotion decisions with low CPU and locking
+ overhead, this table also includes the coarser covering locks and the
+ number of finer-granularity locks they cover.
- * Conflicts are identified by looking for predicate locks
-when tuples are written, and by looking at the MVCC information when
-tuples are read. There is no matching between two RAM-based locks.
+* Conflicts are identified by looking for predicate locks
+ when tuples are written, and by looking at the MVCC information when
+ tuples are read. There is no matching between two RAM-based locks.
- * Because write locks are stored in the heap tuples rather than a
-RAM-based lock table, the optimization described in the Cahill thesis
-which eliminates an SIREAD lock where there is a write lock is
-implemented by the following:
+* Because write locks are stored in the heap tuples rather than a
+ RAM-based lock table, the optimization described in the Cahill thesis
+ which eliminates an SIREAD lock where there is a write lock is
+ implemented by the following:
1. When checking a heap write for conflicts against existing
-predicate locks, a tuple lock on the tuple being written is removed.
+ predicate locks, a tuple lock on the tuple being written is removed.
2. When acquiring a predicate lock on a heap tuple, we
-return quickly without doing anything if it is a tuple written by the
-reading transaction.
-
- * Rather than using conflictIn and conflictOut pointers which use
-NULL to indicate no conflict and a self-reference to indicate
-multiple conflicts or conflicts with committed transactions, we use a
-list of rw-conflicts. With the more complete information, false
-positives are reduced and we have sufficient data for more aggressive
-clean-up and other optimizations:
-
- o We can avoid ever rolling back a transaction until and
-unless there is a pivot where a transaction on the conflict *out*
-side of the pivot committed before either of the other transactions.
-
- o We can avoid ever rolling back a transaction when the
-transaction on the conflict *in* side of the pivot is explicitly or
-implicitly READ ONLY unless the transaction on the conflict *out*
-side of the pivot committed before the READ ONLY transaction acquired
-its snapshot. (An implicit READ ONLY transaction is one which
-committed without writing, even though it was not explicitly declared
-to be READ ONLY.)
-
- o We can more aggressively clean up conflicts, predicate
-locks, and SSI transaction information.
-
- * We allow a READ ONLY transaction to "opt out" of SSI if there are
-no READ WRITE transactions which could cause the READ ONLY
-transaction to ever become part of a "dangerous structure" of
-overlapping transaction dependencies.
-
- * We allow the user to request that a READ ONLY transaction wait
-until the conditions are right for it to start in the "opt out" state
-described above. We add a DEFERRABLE state to transactions, which is
-specified and maintained in a way similar to READ ONLY. It is
-ignored for transactions that are not SERIALIZABLE and READ ONLY.
-
- * When a transaction must be rolled back, we pick among the
-active transactions such that an immediate retry will not fail again
-on conflicts with the same transactions.
-
- * We use the PostgreSQL SLRU system to hold summarized
-information about older committed transactions to put an upper bound
-on RAM used. Beyond that limit, information spills to disk.
-Performance can degrade in a pessimal situation, but it should be
-tolerable, and transactions won't need to be canceled or blocked
-from starting.
+ return quickly without doing anything if it is a tuple written by the
+ reading transaction.
+
+* Rather than using conflictIn and conflictOut pointers which use
+ NULL to indicate no conflict and a self-reference to indicate
+ multiple conflicts or conflicts with committed transactions, we use a
+ list of rw-conflicts. With the more complete information, false
+ positives are reduced and we have sufficient data for more aggressive
+ clean-up and other optimizations:
+
+ - We can avoid ever rolling back a transaction until and
+ unless there is a pivot where a transaction on the conflict *out*
+ side of the pivot committed before either of the other transactions.
+
+ - We can avoid ever rolling back a transaction when the
+ transaction on the conflict *in* side of the pivot is explicitly or
+ implicitly READ ONLY unless the transaction on the conflict *out*
+ side of the pivot committed before the READ ONLY transaction acquired
+ its snapshot. (An implicit READ ONLY transaction is one which
+ committed without writing, even though it was not explicitly declared
+ to be READ ONLY.)
+
+ - We can more aggressively clean up conflicts, predicate
+ locks, and SSI transaction information.
+
+* We allow a READ ONLY transaction to "opt out" of SSI if there are
+ no READ WRITE transactions which could cause the READ ONLY
+ transaction to ever become part of a "dangerous structure" of
+ overlapping transaction dependencies.
+
+* We allow the user to request that a READ ONLY transaction wait
+ until the conditions are right for it to start in the "opt out" state
+ described above. We add a DEFERRABLE state to transactions, which is
+ specified and maintained in a way similar to READ ONLY. It is
+ ignored for transactions that are not SERIALIZABLE and READ ONLY.
+
+* When a transaction must be rolled back, we pick among the
+ active transactions such that an immediate retry will not fail again
+ on conflicts with the same transactions.
+
+* We use the PostgreSQL SLRU system to hold summarized
+ information about older committed transactions to put an upper bound
+ on RAM used. Beyond that limit, information spills to disk.
+ Performance can degrade in a pessimal situation, but it should be
+ tolerable, and transactions won't need to be canceled or blocked
+ from starting.
R&D Issues
@@ -594,34 +594,34 @@ R&D Issues
This is intended to be the place to record specific issues which need
more detailed review or analysis.
- * WAL file replay. While serializable implementations using S2PL
-can guarantee that the write-ahead log contains commits in a sequence
-consistent with some serial execution of serializable transactions,
-SSI cannot make that guarantee. While the WAL replay is no less
-consistent than under snapshot isolation, it is possible that under
-PITR recovery or hot standby a database could reach a readable state
-where some transactions appear before other transactions which would
-have had to precede them to maintain serializable consistency. In
-essence, if we do nothing, WAL replay will be at snapshot isolation
-even for serializable transactions. Is this OK? If not, how do we
-address it?
-
- * External replication. Look at how this impacts external
-replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
-This is related to the "WAL file replay" issue.
-
- * UNIQUE btree search for equality on all columns. Since a search
-of a UNIQUE index using equality tests on all columns will lock the
-heap tuple if an entry is found, it appears that there is no need to
-get a predicate lock on the index in that case. A predicate lock is
-still needed for such a search if a matching index entry which points
-to a visible tuple is not found.
-
- * Minimize touching of shared memory. Should lists in shared
-memory push entries which have just been returned to the front of the
-available list, so they will be popped back off soon and some memory
-might never be touched, or should we keep adding returned items to
-the end of the available list?
+* WAL file replay. While serializable implementations using S2PL
+ can guarantee that the write-ahead log contains commits in a sequence
+ consistent with some serial execution of serializable transactions,
+ SSI cannot make that guarantee. While the WAL replay is no less
+ consistent than under snapshot isolation, it is possible that under
+ PITR recovery or hot standby a database could reach a readable state
+ where some transactions appear before other transactions which would
+ have had to precede them to maintain serializable consistency. In
+ essence, if we do nothing, WAL replay will be at snapshot isolation
+ even for serializable transactions. Is this OK? If not, how do we
+ address it?
+
+* External replication. Look at how this impacts external
+ replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
+ This is related to the "WAL file replay" issue.
+
+* UNIQUE btree search for equality on all columns. Since a search
+ of a UNIQUE index using equality tests on all columns will lock the
+ heap tuple if an entry is found, it appears that there is no need to
+ get a predicate lock on the index in that case. A predicate lock is
+ still needed for such a search if a matching index entry which points
+ to a visible tuple is not found.
+
+* Minimize touching of shared memory. Should lists in shared
+ memory push entries which have just been returned to the front of the
+ available list, so they will be popped back off soon and some memory
+ might never be touched, or should we keep adding returned items to
+ the end of the available list?
References
@@ -638,9 +638,9 @@ http://dx.doi.org/10.1145/1071610.1071615
Architecture of a Database System. Foundations and Trends(R) in
Databases Vol. 1, No. 2 (2007) 141-259.
http://db.cs.berkeley.edu/papers/fntdb07-architecture.pdf
- Of particular interest:
- * 6.1 A Note on ACID
- * 6.2 A Brief Review of Serializability
- * 6.3 Locking and Latching
- * 6.3.1 Transaction Isolation Levels
- * 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
+Of particular interest:
+* 6.1 A Note on ACID
+* 6.2 A Brief Review of Serializability
+* 6.3 Locking and Latching
+* 6.3.1 Transaction Isolation Levels
+* 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README
index 9958d38992..ee14362b8e 100644
--- a/src/backend/utils/fmgr/README
+++ b/src/backend/utils/fmgr/README
@@ -20,18 +20,18 @@ tuple.)
When a function is looked up in pg_proc, the result is represented as
-typedef struct
-{
- PGFunction fn_addr; /* pointer to function or handler to be called */
- Oid fn_oid; /* OID of function (NOT of handler, if any) */
- short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
- bool fn_strict; /* function is "strict" (NULL in => NULL out) */
- bool fn_retset; /* function returns a set (over multiple calls) */
- unsigned char fn_stats; /* collect stats if track_functions > this */
- void *fn_extra; /* extra space for use by handler */
- MemoryContext fn_mcxt; /* memory context to store fn_extra in */
- Node *fn_expr; /* expression parse tree for call, or NULL */
-} FmgrInfo;
+ typedef struct
+ {
+ PGFunction fn_addr; /* pointer to function or handler to be called */
+ Oid fn_oid; /* OID of function (NOT of handler, if any) */
+ short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
+ bool fn_strict; /* function is "strict" (NULL in => NULL out) */
+ bool fn_retset; /* function returns a set (over multiple calls) */
+ unsigned char fn_stats; /* collect stats if track_functions > this */
+ void *fn_extra; /* extra space for use by handler */
+ MemoryContext fn_mcxt; /* memory context to store fn_extra in */
+ Node *fn_expr; /* expression parse tree for call, or NULL */
+ } FmgrInfo;
For an ordinary built-in function, fn_addr is just the address of the C
routine that implements the function. Otherwise it is the address of a
@@ -59,17 +59,17 @@ FmgrInfo than in FunctionCallInfoBaseData where it might more logically go.
During a call of a function, the following data structure is created
and passed to the function:
-typedef struct
-{
- FmgrInfo *flinfo; /* ptr to lookup info used for this call */
- Node *context; /* pass info about context of call */
- Node *resultinfo; /* pass or return extra info about result */
- Oid fncollation; /* collation for function to use */
- bool isnull; /* function must set true if result is NULL */
- short nargs; /* # arguments actually passed */
- NullableDatum args[]; /* Arguments passed to function */
-} FunctionCallInfoBaseData;
-typedef FunctionCallInfoBaseData* FunctionCallInfo;
+ typedef struct
+ {
+ FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+ Node *context; /* pass info about context of call */
+ Node *resultinfo; /* pass or return extra info about result */
+ Oid fncollation; /* collation for function to use */
+ bool isnull; /* function must set true if result is NULL */
+ short nargs; /* # arguments actually passed */
+ NullableDatum args[]; /* Arguments passed to function */
+ } FunctionCallInfoBaseData;
+ typedef FunctionCallInfoBaseData* FunctionCallInfo;
flinfo points to the lookup info used to make the call. Ordinary functions
will probably ignore this field, but function class handlers will need it
@@ -119,11 +119,11 @@ least), and other uglinesses.
Callees, whether they be individual functions or function handlers,
shall always have this signature:
-Datum function (FunctionCallInfo fcinfo);
+ Datum function (FunctionCallInfo fcinfo);
which is represented by the typedef
-typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
+ typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
The function is responsible for setting fcinfo->isnull appropriately
as well as returning a result represented as a Datum. Note that since
@@ -139,11 +139,11 @@ Here are the proposed macros and coding conventions:
The definition of an fmgr-callable function will always look like
-Datum
-function_name(PG_FUNCTION_ARGS)
-{
- ...
-}
+ Datum
+ function_name(PG_FUNCTION_ARGS)
+ {
+ ...
+ }
"PG_FUNCTION_ARGS" just expands to "FunctionCallInfo fcinfo". The main
reason for using this macro is to make it easy for scripts to spot function
@@ -156,26 +156,37 @@ just "fcinfo->args[n].isnull"). It should avoid trying to fetch the value
of any argument that is null.
Both strict and nonstrict functions can return NULL, if needed, with
+
PG_RETURN_NULL();
+
which expands to
+
{ fcinfo->isnull = true; return (Datum) 0; }
Argument values are ordinarily fetched using code like
+
int32 name = PG_GETARG_INT32(number);
For float4, float8, and int8, the PG_GETARG macros will hide whether the
types are pass-by-value or pass-by-reference. For example, if float8 is
pass-by-reference then PG_GETARG_FLOAT8 expands to
+
+
(* (float8 *) DatumGetPointer(fcinfo->args[number].value))
+
and would typically be called like this:
+
float8 arg = PG_GETARG_FLOAT8(0);
+
For what are now historical reasons, the float-related typedefs and macros
express the type width in bytes (4 or 8), whereas we prefer to label the
widths of integer types in bits.
Non-null values are returned with a PG_RETURN_XXX macro of the appropriate
type. For example, PG_RETURN_INT32 expands to
+
return Int32GetDatum(x)
+
PG_RETURN_FLOAT4, PG_RETURN_FLOAT8, and PG_RETURN_INT64 hide whether their
data types are pass-by-value or pass-by-reference, by doing a palloc if
needed.
@@ -290,9 +301,13 @@ transaction cleanup. SQL-callable functions can support this need
using the ErrorSaveContext context mechanism.
To report a "soft" error, a SQL-callable function should call
+
errsave(fcinfo->context, ...)
+
where it would previously have done
+
ereport(ERROR, ...)
+
If the passed "context" is NULL or is not an ErrorSaveContext node,
then errsave behaves precisely as ereport(ERROR): the exception is
thrown via longjmp, so that control does not return. If "context"
@@ -306,7 +321,9 @@ been reported in the ErrorSaveContext node.)
If there is nothing to do except return after calling errsave(),
you can save a line or two by writing
+
ereturn(fcinfo->context, dummy_value, ...)
+
to perform errsave() and then "return dummy_value".
An error reported "softly" must be safe, in the sense that there is
diff --git a/src/backend/utils/mb/README b/src/backend/utils/mb/README
index 4447881ead..afbe4c65d1 100644
--- a/src/backend/utils/mb/README
+++ b/src/backend/utils/mb/README
@@ -3,17 +3,17 @@ src/backend/utils/mb/README
Encodings
=========
-conv.c: static functions and a public table for code conversion
-mbutils.c: public functions for the backend only.
-stringinfo_mb.c: public backend-only multibyte-aware stringinfo functions
-wstrcmp.c: strcmp for mb
-wstrncmp.c: strncmp for mb
+ conv.c static functions and a public table for code conversion
+ mbutils.c public functions for the backend only.
+ stringinfo_mb.c public backend-only multibyte-aware stringinfo functions
+ wstrcmp.c strcmp for mb
+ wstrncmp.c strncmp for mb
See also in src/common/:
-encnames.c: public functions for encoding names
-wchar.c: mostly static functions and a public table for mb string and
- multibyte conversion
+ encnames.c public functions for encoding names
+ wchar.c mostly static functions and a public table for mb string and
+ multibyte conversion
Introduction
------------
diff --git a/src/backend/utils/misc/README b/src/backend/utils/misc/README
index 85d97d29b6..abfa473757 100644
--- a/src/backend/utils/misc/README
+++ b/src/backend/utils/misc/README
@@ -23,29 +23,37 @@ modify the default SHOW display for a variable.
If a check_hook is provided, it points to a function of the signature
+
bool check_hook(datatype *newvalue, void **extra, GucSource source)
+
The "newvalue" argument is of type bool *, int *, double *, or char **
for bool, int/enum, real, or string variables respectively. The check
function should validate the proposed new value, and return true if it is
OK or false if not. The function can optionally do a few other things:
* When rejecting a bad proposed value, it may be useful to append some
-additional information to the generic "invalid value for parameter FOO"
-complaint that guc.c will emit. To do that, call
+ additional information to the generic "invalid value for parameter FOO"
+ complaint that guc.c will emit. To do that, call
+
void GUC_check_errdetail(const char *format, ...)
-where the format string and additional arguments follow the rules for
-errdetail() arguments. The resulting string will be emitted as the
-DETAIL line of guc.c's error report, so it should follow the message style
-guidelines for DETAIL messages. There is also
+
+ where the format string and additional arguments follow the rules for
+ errdetail() arguments. The resulting string will be emitted as the
+ DETAIL line of guc.c's error report, so it should follow the message style
+ guidelines for DETAIL messages. There is also
+
void GUC_check_errhint(const char *format, ...)
-which can be used in the same way to append a HINT message.
-Occasionally it may even be appropriate to override guc.c's generic primary
-message or error code, which can be done with
+
+ which can be used in the same way to append a HINT message.
+ Occasionally it may even be appropriate to override guc.c's generic primary
+ message or error code, which can be done with
+
void GUC_check_errcode(int sqlerrcode)
void GUC_check_errmsg(const char *format, ...)
-In general, check_hooks should avoid throwing errors directly if possible,
-though this may be impractical to avoid for some corner cases such as
-out-of-memory.
+
+ In general, check_hooks should avoid throwing errors directly if possible,
+ though this may be impractical to avoid for some corner cases such as
+ out-of-memory.
* Since the newvalue is pass-by-reference, the function can modify it.
This might be used for example to canonicalize the spelling of a string
@@ -76,7 +84,9 @@ assignment will occur.
If an assign_hook is provided, it points to a function of the signature
+
void assign_hook(datatype newvalue, void *extra)
+
where the type of "newvalue" matches the kind of variable, and "extra"
is the derived-information pointer returned by the check_hook (always
NULL if there is no check_hook). This function is called immediately
@@ -110,7 +120,9 @@ needing to check GUC values outside a transaction.
If a show_hook is provided, it points to a function of the signature
+
const char *show_hook(void)
+
This hook allows variable-specific computation of the value displayed
by SHOW (and other SQL features for showing GUC variable values).
The return value can point to a static buffer, since show functions are
@@ -214,23 +226,23 @@ The merged entry will have level N-1 and prior = older prior, so easiest
to keep older entry and free newer. There are 12 possibilities since
we already handled level N state = SAVE:
-N-1 N
+ N-1 N
-SAVE SET discard top prior, set state SET
-SAVE LOCAL discard top prior, no change to stack entry
-SAVE SET+LOCAL discard top prior, copy masked, state S+L
+ SAVE SET discard top prior, set state SET
+ SAVE LOCAL discard top prior, no change to stack entry
+ SAVE SET+LOCAL discard top prior, copy masked, state S+L
-SET SET discard top prior, no change to stack entry
-SET LOCAL copy top prior to masked, state S+L
-SET SET+LOCAL discard top prior, copy masked, state S+L
+ SET SET discard top prior, no change to stack entry
+ SET LOCAL copy top prior to masked, state S+L
+ SET SET+LOCAL discard top prior, copy masked, state S+L
-LOCAL SET discard top prior, set state SET
-LOCAL LOCAL discard top prior, no change to stack entry
-LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ LOCAL SET discard top prior, set state SET
+ LOCAL LOCAL discard top prior, no change to stack entry
+ LOCAL SET+LOCAL discard top prior, copy masked, state S+L
-SET+LOCAL SET discard top prior and second masked, state SET
-SET+LOCAL LOCAL discard top prior, no change to stack entry
-SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ SET+LOCAL SET discard top prior and second masked, state SET
+ SET+LOCAL LOCAL discard top prior, no change to stack entry
+ SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
RESET is executed like a SET, but using the reset_val as the desired new
diff --git a/src/backend/utils/mmgr/README b/src/backend/utils/mmgr/README
index 695088bb66..27a49a9a36 100644
--- a/src/backend/utils/mmgr/README
+++ b/src/backend/utils/mmgr/README
@@ -412,11 +412,11 @@ GetMemoryChunkMethodID() and finding the corresponding MemoryContextMethods
in the mcxt_methods[] array. For convenience, the MCXT_METHOD() macro is
provided, making the code as simple as:
-void
-pfree(void *pointer)
-{
- MCXT_METHOD(pointer, free_p)(pointer);
-}
+ void
+ pfree(void *pointer)
+ {
+ MCXT_METHOD(pointer, free_p)(pointer);
+ }
All of the current memory contexts make use of the MemoryChunk header type
which is defined in memutils_memorychunk.h. This suits all of the existing
diff --git a/src/backend/utils/resowner/README b/src/backend/utils/resowner/README
index cbf34e0b56..d0b4eb1d72 100644
--- a/src/backend/utils/resowner/README
+++ b/src/backend/utils/resowner/README
@@ -83,13 +83,13 @@ references, to name a few examples.
To add a new kind of resource, define a ResourceOwnerDesc to describe it.
For example:
-static const ResourceOwnerDesc myresource_desc = {
- .name = "My fancy resource",
- .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
- .release_priority = RELEASE_PRIO_FIRST,
- .ReleaseResource = ReleaseMyResource,
- .DebugPrint = PrintMyResource
-};
+ static const ResourceOwnerDesc myresource_desc = {
+ .name = "My fancy resource",
+ .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+ .release_priority = RELEASE_PRIO_FIRST,
+ .ReleaseResource = ReleaseMyResource,
+ .DebugPrint = PrintMyResource
+ };
ResourceOwnerRemember() and ResourceOwnerForget() functions take a pointer
to that struct, along with a Datum to represent the resource. The meaning
@@ -139,28 +139,28 @@ within each phase.
For example, imagine that you have two ResourceOwners, parent and child,
as follows:
-Parent
- parent resource BEFORE_LOCKS priority 1
- parent resource BEFORE_LOCKS priority 2
- parent resource AFTER_LOCKS priority 10001
- parent resource AFTER_LOCKS priority 10002
- Child
- child resource BEFORE_LOCKS priority 1
- child resource BEFORE_LOCKS priority 2
- child resource AFTER_LOCKS priority 10001
- child resource AFTER_LOCKS priority 10002
+ Parent
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
+ Child
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
These resources would be released in the following order:
-child resource BEFORE_LOCKS priority 1
-child resource BEFORE_LOCKS priority 2
-parent resource BEFORE_LOCKS priority 1
-parent resource BEFORE_LOCKS priority 2
-(locks)
-child resource AFTER_LOCKS priority 10001
-child resource AFTER_LOCKS priority 10002
-parent resource AFTER_LOCKS priority 10001
-parent resource AFTER_LOCKS priority 10002
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ (locks)
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
To release all the resources, you need to call ResourceOwnerRelease() three
times, once for each phase. You may perform additional tasks between the
diff --git a/src/interfaces/ecpg/preproc/README.parser b/src/interfaces/ecpg/preproc/README.parser
index ddc3061d48..ec517a3b5a 100644
--- a/src/interfaces/ecpg/preproc/README.parser
+++ b/src/interfaces/ecpg/preproc/README.parser
@@ -14,29 +14,38 @@ ECPG modifies and extends the core grammar in a way that
actions for grammar rules.
In "ecpg.addons", every modified rule follows this pattern:
+
ECPG: dumpedtokens postfix
+
where "dumpedtokens" is simply tokens from core gram.y's
rules concatenated together. e.g. if gram.y has this:
- ruleA: tokenA tokenB tokenC {...}
+
+ ruleA: tokenA tokenB tokenC {...}
+
then "dumpedtokens" is "ruleAtokenAtokenBtokenC".
"postfix" above can be:
+
a) "block" - the automatic rule created by parse.pl is completely
- overridden, the code block has to be written completely as
- it were in a plain bison grammar
+ overridden, the code block has to be written completely as
+ it were in a plain bison grammar
b) "rule" - the automatic rule is extended on, so new syntaxes
- are accepted for "ruleA". E.g.:
+ are accepted for "ruleA". E.g.:
+
ECPG: ruleAtokenAtokenBtokenC rule
| tokenD tokenE { action_code; }
...
+
It will be substituted with:
+
ruleA: <original syntax forms and actions up to and including
"tokenA tokenB tokenC">
| tokenD tokenE { action_code; }
...
+
c) "addon" - the automatic action for the rule (SQL syntax constructed
- from the tokens concatenated together) is prepended with a new
- action code part. This code part is written as is's already inside
- the { ... }
+ from the tokens concatenated together) is prepended with a new
+ action code part. This code part is written as is's already inside
+ the { ... }
Multiple "addon" or "block" lines may appear together with the
new code block if the code block is common for those rules.
diff --git a/src/port/README b/src/port/README
index ed5c54a72f..e5aeed07b6 100644
--- a/src/port/README
+++ b/src/port/README
@@ -14,7 +14,7 @@ libraries. This is done by removing -lpgport from the link line:
# Need to recompile any libpgport object files
LIBS := $(filter-out -lpgport, $(LIBS))
-and adding infrastructure to recompile the object files:
+ and adding infrastructure to recompile the object files:
OBJS= execute.o typename.o descriptor.o data.o error.o prepare.o memory.o \
connect.o misc.o path.o exec.o \
diff --git a/src/test/isolation/README b/src/test/isolation/README
index 5818ca5003..471b0a5029 100644
--- a/src/test/isolation/README
+++ b/src/test/isolation/README
@@ -12,23 +12,33 @@ serializable isolation level; but tests for other sorts of concurrent
behaviors have been added as well.
You can run the tests against the current build tree by typing
+
make check
+
Alternatively, you can run against an existing installation by typing
+
make installcheck
+
(This will contact a server at the default port expected by libpq.
You can set PGPORT and so forth in your environment to control this.)
To run just specific test(s) against an installed server,
you can do something like
+
./pg_isolation_regress fk-contention fk-deadlock
+
(look into the specs/ subdirectory to see the available tests).
Certain tests require the server's max_prepared_transactions parameter to be
set to at least 3; therefore they are not run by default. To include them in
the test run, use
+
make check-prepared-txns
+
or
+
make installcheck-prepared-txns
+
after making sure the server configuration is correct (see TEMP_CONFIG
to adjust this in the "check" case).
@@ -64,7 +74,7 @@ that are to be run.
A test specification consists of four parts, in this order:
-setup { <SQL> }
+setup { < SQL > }
The given SQL block is executed once (per permutation) before running
the test. Create any test tables or other required objects here. This
@@ -74,13 +84,13 @@ setup { <SQL> }
and some statements such as VACUUM cannot be combined with others in such
a block.)
-teardown { <SQL> }
+teardown { < SQL > }
The teardown SQL block is executed once after the test is finished. Use
this to clean up in preparation for the next permutation, e.g dropping
any test tables created by setup. This part is optional.
-session <name>
+session < name >
There are normally several "session" parts in a spec file. Each
session is executed in its own connection. A session part consists
@@ -91,13 +101,13 @@ session <name>
Each step has the syntax
- step <name> { <SQL> }
+ step < name > { < SQL > }
- where <name> is a name identifying this step, and <SQL> is a SQL statement
+ where < name > is a name identifying this step, and < SQL > is a SQL statement
(or statements, separated by semicolons) that is executed in the step.
Step names must be unique across the whole spec file.
-permutation <step name> ...
+permutation < step name > ...
A permutation line specifies a list of steps that are run in that order.
Any number of permutation lines can appear. If no permutation lines are
@@ -116,10 +126,10 @@ whether you quote them or not. You must use quotes if you want to use
an isolation test keyword (such as "permutation") as a name.
A # character begins a comment, which extends to the end of the line.
-(This does not work inside <SQL> blocks, however. Use the usual SQL
+(This does not work inside < SQL > blocks, however. Use the usual SQL
comment conventions there.)
-There is no way to include a "}" character in an <SQL> block.
+There is no way to include a "}" character in an < SQL > block.
For each permutation of the session steps (whether these are manually
specified in the spec file, or automatically generated), the isolation
@@ -187,9 +197,9 @@ step has completed. (If the other step is used more than once in the
current permutation, this step cannot complete while any of those
instances is active.)
-A marker of the form "<other step name> notices <n>" (where <n> is a
+A marker of the form "< other step name > notices < n >" (where < n > is a
positive integer) indicates that this step may not be reported as
-completing until the other step's session has returned at least <n>
+completing until the other step's session has returned at least < n >
NOTICE messages, counting from when this step is launched. This is useful
for stabilizing cases where a step can return NOTICE messages before it
actually completes, and those messages must be synchronized with the
diff --git a/src/test/kerberos/README b/src/test/kerberos/README
index a048d442af..dc53747fd9 100644
--- a/src/test/kerberos/README
+++ b/src/test/kerberos/README
@@ -23,9 +23,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/dblink and contrib/postgres_fdw in addition to the core code.
Run
+
make check PG_TEST_EXTRA=kerberos
+
or
+
make installcheck PG_TEST_EXTRA=kerberos
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/locale/README b/src/test/locale/README
index e290e31480..eb6a7c6124 100644
--- a/src/test/locale/README
+++ b/src/test/locale/README
@@ -9,8 +9,11 @@ locale data. Then there are test-sort.pl and test-sort.py that test
collating.
To run a test for some locale run
+
make check-$locale
+
for example
+
make check-koi8-r
Currently, there are only tests for a few locales available. The script
diff --git a/src/test/modules/dummy_seclabel/README b/src/test/modules/dummy_seclabel/README
index a3fcbd7599..1a12fdaad2 100644
--- a/src/test/modules/dummy_seclabel/README
+++ b/src/test/modules/dummy_seclabel/README
@@ -18,13 +18,13 @@ Usage
Here's a simple example of usage:
-# postgresql.conf
-shared_preload_libraries = 'dummy_seclabel'
+ # postgresql.conf
+ shared_preload_libraries = 'dummy_seclabel'
-postgres=# CREATE TABLE t (a int, b text);
-CREATE TABLE
-postgres=# SECURITY LABEL ON TABLE t IS 'classified';
-SECURITY LABEL
+ postgres=# CREATE TABLE t (a int, b text);
+ CREATE TABLE
+ postgres=# SECURITY LABEL ON TABLE t IS 'classified';
+ SECURITY LABEL
The dummy_seclabel module provides only four hardcoded
labels: unclassified, classified,
diff --git a/src/test/modules/test_parser/README b/src/test/modules/test_parser/README
index 0a11ec85fb..735bfe9a00 100644
--- a/src/test/modules/test_parser/README
+++ b/src/test/modules/test_parser/README
@@ -5,12 +5,12 @@ a starting point for developing your own parser.
test_parser recognizes words separated by white space,
and returns just two token types:
-mydb=# SELECT * FROM ts_token_type('testparser');
- tokid | alias | description
--------+-------+---------------
- 3 | word | Word
- 12 | blank | Space symbols
-(2 rows)
+ mydb=# SELECT * FROM ts_token_type('testparser');
+ tokid | alias | description
+ -------+-------+---------------
+ 3 | word | Word
+ 12 | blank | Space symbols
+ (2 rows)
These token numbers have been chosen to be compatible with the default
parser's numbering. This allows us to use its headline()
@@ -24,38 +24,38 @@ parser testparser. It has no user-configurable parameters.
You can test the parser with, for example,
-mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
- tokid | token
--------+--------
- 3 | That's
- 12 |
- 3 | my
- 12 |
- 3 | first
- 12 |
- 3 | own
- 12 |
- 3 | parser
+ mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
+ tokid | token
+ -------+--------
+ 3 | That's
+ 12 |
+ 3 | my
+ 12 |
+ 3 | first
+ 12 |
+ 3 | own
+ 12 |
+ 3 | parser
Real-world use requires setting up a text search configuration
that uses the parser. For example,
-mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
-CREATE TEXT SEARCH CONFIGURATION
-
-mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
-mydb-# ADD MAPPING FOR word WITH english_stem;
-ALTER TEXT SEARCH CONFIGURATION
-
-mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
- to_tsvector
--------------------------------
- 'that':1 'first':3 'parser':5
-(1 row)
-
-mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
-mydb(# to_tsquery('testcfg', 'star'));
- ts_headline
------------------------------------------------------------------
- Supernovae <b>stars</b> are the brightest phenomena in galaxies
-(1 row)
+ mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
+ CREATE TEXT SEARCH CONFIGURATION
+
+ mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
+ mydb-# ADD MAPPING FOR word WITH english_stem;
+ ALTER TEXT SEARCH CONFIGURATION
+
+ mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
+ to_tsvector
+ -------------------------------
+ 'that':1 'first':3 'parser':5
+ (1 row)
+
+ mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
+ mydb(# to_tsquery('testcfg', 'star'));
+ ts_headline
+ -----------------------------------------------------------------
+ Supernovae <b>stars</b> are the brightest phenomena in galaxies
+ (1 row)
diff --git a/src/test/modules/test_regex/README b/src/test/modules/test_regex/README
index 3ef152d4e1..7da5e67a9c 100644
--- a/src/test/modules/test_regex/README
+++ b/src/test/modules/test_regex/README
@@ -5,7 +5,7 @@ aren't currently exposed at the SQL level by PostgreSQL.
Currently, one function is provided:
-test_regex(pattern text, string text, flags text) returns setof text[]
+ test_regex(pattern text, string text, flags text) returns setof text[]
Reports an error if the pattern is an invalid regex. Otherwise,
the first row of output contains the number of subexpressions,
diff --git a/src/test/modules/test_rls_hooks/README b/src/test/modules/test_rls_hooks/README
index c22e0d3fb4..b561bb0192 100644
--- a/src/test/modules/test_rls_hooks/README
+++ b/src/test/modules/test_rls_hooks/README
@@ -3,14 +3,14 @@ define additional policies to be used.
Functions
=========
-test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with OR.
-test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with AND.
diff --git a/src/test/modules/test_shm_mq/README b/src/test/modules/test_shm_mq/README
index 641407bee0..1df3d38bde 100644
--- a/src/test/modules/test_shm_mq/README
+++ b/src/test/modules/test_shm_mq/README
@@ -14,9 +14,9 @@ Functions
=========
-test_shm_mq(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1)
- RETURNS void
+ test_shm_mq(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1)
+ RETURNS void
This function sends and receives messages synchronously. The user
backend sends the provided message to the first background worker using
@@ -31,10 +31,10 @@ the user backend verifies that the message finally received matches the
one originally sent and throws an error if not.
-test_shm_mq_pipelined(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1,
- verify bool default true)
- RETURNS void
+ test_shm_mq_pipelined(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1,
+ verify bool default true)
+ RETURNS void
This function sends the same message multiple times, as specified by the
repeat count, to the first background worker using a queue of the given
diff --git a/src/test/recovery/README b/src/test/recovery/README
index 896df0ad05..c18fe7e21e 100644
--- a/src/test/recovery/README
+++ b/src/test/recovery/README
@@ -14,9 +14,13 @@ contrib/pg_prewarm, contrib/pg_stat_statements and contrib/test_decoding
in addition to the core code.
Run
- make check
+
+ make check
+
or
- make installcheck
+
+ make installcheck
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/ssl/README b/src/test/ssl/README
index 2101a466d2..957694c689 100644
--- a/src/test/ssl/README
+++ b/src/test/ssl/README
@@ -20,9 +20,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/sslinfo in addition to the core code.
Run
+
make check PG_TEST_EXTRA=ssl
+
or
+
make installcheck PG_TEST_EXTRA=ssl
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
@@ -39,49 +43,49 @@ Certificates
The test suite needs a set of public/private key pairs and certificates to
run:
-root_ca
+* root_ca
root CA, use to sign the server and client CA certificates.
-server_ca
+* server_ca
CA used to sign server certificates.
-client_ca
+* client_ca
CA used to sign client certificates.
-server-cn-only
+* server-cn-only
server-cn-and-alt-names
server-single-alt-name
server-multiple-alt-names
-server-no-names
+server-no-names:
server certificates, with small variations in the hostnames present
in the certificate. Signed by server_ca.
-server-password
+* server-password:
same as server-cn-only, but password-protected.
-client
+* client:
a client certificate, for user "ssltestuser". Signed by client_ca.
-client-revoked
+* client-revoked:
like "client", but marked as revoked in the client CA's CRL.
In addition, there are a few files that combine various certificates together
in the same file:
-both-cas-1
+* both-cas-1:
Contains root_ca.crt, client_ca.crt and server_ca.crt, in that order.
-both-cas-2
+* both-cas-2:
Contains root_ca.crt, server_ca.crt and client_ca.crt, in that order.
-root+server_ca
+* root+server_ca:
Contains root_crt and server_ca.crt. For use as client's "sslrootcert"
option.
-root+client_ca
+* root+client_ca:
Contains root_crt and client_ca.crt. For use as server's "ssl_ca_file".
-client+client_ca
+* client+client_ca:
Contains client.crt and client_ca.crt in that order. For use as client's
certificate chain.
diff --git a/src/timezone/tznames/README b/src/timezone/tznames/README
index 6d355e4616..3e05c80579 100644
--- a/src/timezone/tznames/README
+++ b/src/timezone/tznames/README
@@ -12,7 +12,7 @@ be a file here that serves your needs. If not, you can create your own.
In order to use one of these files, you need to set
- timezone_abbreviations = 'xyz'
+ timezone_abbreviations = 'xyz'
in any of the usual ways for setting a parameter, where xyz is the filename
that contains the desired time zone abbreviations.
diff --git a/src/tools/ci/README b/src/tools/ci/README
index 30ddd200c9..e82073ff8a 100644
--- a/src/tools/ci/README
+++ b/src/tools/ci/README
@@ -35,7 +35,7 @@ See also https://cirrus-ci.org/guide/quick-start/
Once enabled on a repository, future commits and pull-requests in that
repository will automatically trigger CI builds. These are visible from the
commit history / PRs, and can also be viewed in the cirrus-ci UI at
-https://cirrus-ci.com/github/<username>/<reponame>/
+https://cirrus-ci.com/github/< username >/< reponame >/
Hint: all build log files are uploaded to cirrus-ci and can be downloaded
from the "Artifacts" section from the cirrus-ci UI after clicking into a
@@ -74,7 +74,7 @@ When running a lot of tests in a repository, cirrus-ci's free credits do not
suffice. In those cases a repository can be configured to use other
infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
variable can be configured for the repository in the cirrus-ci web interface,
-at https://cirrus-ci.com/github/<user or organization>. The file referenced
+at https://cirrus-ci.com/github/< user or organization >. The file referenced
(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
overwrite the default execution method for different operating systems,
defined in .cirrus.yml, by redefining the relevant yaml anchors.
diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README
index b6cd4c6f6b..d984af8f1d 100644
--- a/src/tools/pgindent/README
+++ b/src/tools/pgindent/README
@@ -10,7 +10,7 @@ http://adpgtech.blogspot.com/2015/05/running-pgindent-on-non-core-code-or.html
PREREQUISITES:
-
+--------------
1) Install pg_bsd_indent in your PATH. Its source code is in the
sibling directory src/tools/pg_bsd_indent; see the directions
in that directory's README file.
@@ -22,13 +22,16 @@ PREREQUISITES:
To install, follow the usual install process for a Perl module
("man perlmodinstall" explains it). Or, if you have cpan installed,
this should work:
+
cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+
Or if you have cpanm installed, you can just use:
+
cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
-
+--------------------------------------------
1) Change directory to the top of the source tree.
2) Run pgindent on the C files:
@@ -64,7 +67,7 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
AT LEAST ONCE PER RELEASE CYCLE:
-
+--------------------------------
1) Download the latest typedef file from the buildfarm:
wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
--
2.39.3 (Apple Git-146)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-09-10 15:37 Tom Lane <[email protected]>
parent: Daniel Gustafsson <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Tom Lane @ 2024-09-10 15:37 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; PostgreSQL Developers <[email protected]>
Daniel Gustafsson <[email protected]> writes:
> Since there doesn't seem to be much interest in going all the way to Markdown,
> the attached 0001 is just the formatting changes for achieving (to some degree)
> consistency among the README's. This mostly boils down to using a consistent
> amount of whitespace around code, using the same indentation on bullet lists
> and starting sections the same way. Inspecting the patch with git diff -w
> reveals that it's not much left once whitespace is ignored. There might be a
> few markdown hunks left which I'll hunt down in case anyone is interested in
> this.
> As an added bonus this still makes most READMEs render nicely as Markdown, just
> not automatically on Github as it doesn't know the filetype.
I did not inspect the patch in detail, but this approach seems
like a reasonable compromise. However, if we're not officially
going to Markdown, how likely is it that these files will
stay valid in future edits? I suspect most of us don't have
those syntax rules wired into our fingers (I sure don't).
regards, tom lane
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-09-10 18:25 Daniel Gustafsson <[email protected]>
parent: Tom Lane <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Daniel Gustafsson @ 2024-09-10 18:25 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; PostgreSQL Developers <[email protected]>
> On 10 Sep 2024, at 17:37, Tom Lane <[email protected]> wrote:
>
> Daniel Gustafsson <[email protected]> writes:
>> Since there doesn't seem to be much interest in going all the way to Markdown,
>> the attached 0001 is just the formatting changes for achieving (to some degree)
>> consistency among the README's. This mostly boils down to using a consistent
>> amount of whitespace around code, using the same indentation on bullet lists
>> and starting sections the same way. Inspecting the patch with git diff -w
>> reveals that it's not much left once whitespace is ignored. There might be a
>> few markdown hunks left which I'll hunt down in case anyone is interested in
>> this.
>
>> As an added bonus this still makes most READMEs render nicely as Markdown, just
>> not automatically on Github as it doesn't know the filetype.
>
> I did not inspect the patch in detail, but this approach seems
> like a reasonable compromise. However, if we're not officially
> going to Markdown, how likely is it that these files will
> stay valid in future edits? I suspect most of us don't have
> those syntax rules wired into our fingers (I sure don't).
I'm not too worried, especially since we're not making any guarantees about
conforming to a set syntax. We had written more or less correct Markdown
already, if we continue to create new content in the style of the surrounding
existing content then I'm confident they'll stay very close to markdown.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-09-23 11:58 Peter Eisentraut <[email protected]>
parent: Daniel Gustafsson <[email protected]>
1 sibling, 1 reply; 33+ messages in thread
From: Peter Eisentraut @ 2024-09-23 11:58 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
On 10.09.24 14:50, Daniel Gustafsson wrote:
>> On 1 Jul 2024, at 12:22, Daniel Gustafsson <[email protected]> wrote:
>
>> Attached is a v2 which fixes a conflict, if there is no interest in Markdown
>> I'll drop 0001 and the markdown-specifics from 0002 to instead target increased
>> consistency.
>
> Since there doesn't seem to be much interest in going all the way to Markdown,
> the attached 0001 is just the formatting changes for achieving (to some degree)
> consistency among the README's. This mostly boils down to using a consistent
> amount of whitespace around code, using the same indentation on bullet lists
> and starting sections the same way. Inspecting the patch with git diff -w
> reveals that it's not much left once whitespace is ignored. There might be a
> few markdown hunks left which I'll hunt down in case anyone is interested in
> this.
I went through this file by file and checked the results of a
markdown-to-HTML conversion using cmark and looking at the raw output
source files.
A lot of the changes are obvious and make sense. But there are a number
of cases of code within lists or nested lists or both that need further
careful investigation. I'm attaching a fixup patch where I tried to
improve some of this (and a few other things I found along the way).
Some of the more complicated ones, such as
src/backend/storage/lmgr/README-SSI, will need to be checked again and
even more carefully to make sure that the meaning is not altered by
these patches.
One underlying problem that I see is that markdown assumes four-space
tabs, but a standard editor configuration (and apparently your editor)
uses 8 tabs. But then, if you have a common situation like
```
1. Run this code
<tab>$ sudo kill
```
then that's incorrect (the code line will not be inside the list),
because it should be
```
1. Run this code
<tab><tab>$ sudo kill
```
or
```
1. Run this code
<8 spaces>$ sudo kill
```
So we need to think about a way to make this more robust for future
people editing. Maybe something in .gitattributes or some editor
settings. Otherwise, it will be all over the places after a while.
(There are also a couple of places where apparently you changed
whitespace that wasn't necessary to be changed.)
Apart from this, I don't changing the placeholders like <foo> to < foo
>. In some cases, this really decreases readability. Maybe we should
look for different approaches there.
Maybe there are some easy changes that could be extracted from this
patch, but the whitespace and list issue needs more consideration.
From 990329d7a8e6110ecf099162680f6c0eaf38ea14 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Mon, 23 Sep 2024 13:13:18 +0200
Subject: [PATCH] fixup! Standardize syntax in internal documentation
---
contrib/start-scripts/macos/README | 6 ++--
src/backend/access/gin/README | 6 ++--
src/backend/access/gist/README | 8 ++---
src/backend/access/hash/README | 2 +-
src/backend/lib/README | 22 ++++++------
src/backend/optimizer/plan/README | 6 ++--
src/backend/parser/README | 42 +++++++++++------------
src/backend/storage/lmgr/README-SSI | 6 ++--
src/backend/utils/fmgr/README | 1 -
src/backend/utils/misc/README | 8 ++---
src/interfaces/ecpg/preproc/README.parser | 20 +++++------
src/port/README | 8 ++---
src/test/recovery/README | 4 +--
src/test/ssl/README | 14 ++++----
src/tools/pgindent/README | 24 ++++++-------
15 files changed, 88 insertions(+), 89 deletions(-)
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README
index 8fe6efb657d..c71f98880ca 100644
--- a/contrib/start-scripts/macos/README
+++ b/contrib/start-scripts/macos/README
@@ -15,11 +15,11 @@ if you plan to run the Postgres server under some user name other
than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
- /Library/LaunchDaemons/. You must do this as root:
+/Library/LaunchDaemons/. You must do this as root:
- sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
+ sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
- because the file will be ignored if it is not root-owned.
+ because the file will be ignored if it is not root-owned.
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README
index 08faa944105..ccff6896513 100644
--- a/src/backend/access/gin/README
+++ b/src/backend/access/gin/README
@@ -124,9 +124,9 @@ know are:
or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
the "category" of the null entry. These are the possible categories:
- 1 = ordinary null key value extracted from an indexable item
- 2 = placeholder for zero-key indexable item
- 3 = placeholder for null indexable item
+ 1 = ordinary null key value extracted from an indexable item
+ 2 = placeholder for zero-key indexable item
+ 3 = placeholder for null indexable item
Placeholder null entries are inserted into the index because otherwise
there would be no index entry at all for an empty or null indexable item,
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README
index af082fc2bb0..62818b09d79 100644
--- a/src/backend/access/gist/README
+++ b/src/backend/access/gist/README
@@ -477,7 +477,7 @@ value. The page is not recycled, until that XID is no longer visible to
anyone. That's much more conservative than necessary, but let's keep it
simple.
-
-Authors:
- Teodor Sigaev <[email protected]>
- Oleg Bartunov <[email protected]>
+Authors
+-------
+* Teodor Sigaev <[email protected]>
+* Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README
index a5df13a68a2..5b4c758df62 100644
--- a/src/backend/access/hash/README
+++ b/src/backend/access/hash/README
@@ -255,7 +255,7 @@ The reader algorithm is:
release the buffer content lock on old bucket, but not pin
retake the buffer content lock on new bucket
arrange to scan the old bucket normally and the new bucket for
- tuples which are not moved-by-split
+ tuples which are not moved-by-split
-- then, per read request:
reacquire content lock on current page
step to next page if necessary (no chaining of content locks, but keep
diff --git a/src/backend/lib/README b/src/backend/lib/README
index fc8e1aa1f7c..735e3790513 100644
--- a/src/backend/lib/README
+++ b/src/backend/lib/README
@@ -1,27 +1,27 @@
This directory contains a general purpose data structures, for use anywhere
in the backend:
- binaryheap.c - a binary heap
+* binaryheap.c - a binary heap
- bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
+* bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
- bloomfilter.c - probabilistic, space-efficient set membership testing
+* bloomfilter.c - probabilistic, space-efficient set membership testing
- dshash.c - concurrent hash tables backed by dynamic shared memory areas
+* dshash.c - concurrent hash tables backed by dynamic shared memory areas
- hyperloglog.c - a streaming cardinality estimator
+* hyperloglog.c - a streaming cardinality estimator
- ilist.c - single and double-linked lists
+* ilist.c - single and double-linked lists
- integerset.c - a data structure for holding large set of integers
+* integerset.c - a data structure for holding large set of integers
- knapsack.c - knapsack problem solver
+* knapsack.c - knapsack problem solver
- pairingheap.c - a pairing heap
+* pairingheap.c - a pairing heap
- rbtree.c - a red-black tree
+* rbtree.c - a red-black tree
- stringinfo.c - an extensible string type
+* stringinfo.c - an extensible string type
Aside from the inherent characteristics of the data structures, there are a
diff --git a/src/backend/optimizer/plan/README b/src/backend/optimizer/plan/README
index 93dd422dc15..04d9c50c51e 100644
--- a/src/backend/optimizer/plan/README
+++ b/src/backend/optimizer/plan/README
@@ -91,9 +91,9 @@ on each call) ExecReScan() now supports most of Plan types...
Explanation of EXPLAIN.
-vac=> explain select * from tmp where x >= (select max(x2) from test2
-where y2 = y and exists (select * from tempx where tx = x));
-NOTICE: QUERY PLAN:
+ vac=> explain select * from tmp where x >= (select max(x2) from test2
+ where y2 = y and exists (select * from tempx where tx = x));
+ NOTICE: QUERY PLAN:
Seq Scan on tmp (cost=40.03 size=101 width=8)
SubPlan
diff --git a/src/backend/parser/README b/src/backend/parser/README
index e6016fa4301..9ac8026c73a 100644
--- a/src/backend/parser/README
+++ b/src/backend/parser/README
@@ -7,27 +7,27 @@ This directory does more than tokenize and parse SQL queries. It also
creates Query structures for the various complex queries that are passed
to the optimizer and then executor.
- parser.c things start here
- scan.l break query into tokens
- scansup.c handle escapes in input strings
- gram.y parse the tokens and produce a "raw" parse tree
- analyze.c top level of parse analysis for optimizable queries
- parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
- parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
- parse_coerce.c handle coercing expressions to different data types
- parse_collate.c assign collation information in completed expressions
- parse_cte.c handle Common Table Expressions (WITH clauses)
- parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
- parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
- parse_func.c handle functions, table.column and column identifiers
- parse_merge.c handle MERGE
- parse_node.c create nodes for various structures
- parse_oper.c handle operators in expressions
- parse_param.c handle Params (for the cases used in the core backend)
- parse_relation.c support routines for tables and column handling
- parse_target.c handle the result list of the query
- parse_type.c support routines for data type handling
- parse_utilcmd.c parse analysis for utility commands (done at execution time)
+* parser.c things start here
+* scan.l break query into tokens
+* scansup.c handle escapes in input strings
+* gram.y parse the tokens and produce a "raw" parse tree
+* analyze.c top level of parse analysis for optimizable queries
+* parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
+* parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
+* parse_coerce.c handle coercing expressions to different data types
+* parse_collate.c assign collation information in completed expressions
+* parse_cte.c handle Common Table Expressions (WITH clauses)
+* parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
+* parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
+* parse_func.c handle functions, table.column and column identifiers
+* parse_merge.c handle MERGE
+* parse_node.c create nodes for various structures
+* parse_oper.c handle operators in expressions
+* parse_param.c handle Params (for the cases used in the core backend)
+* parse_relation.c support routines for tables and column handling
+* parse_target.c handle the result list of the query
+* parse_type.c support routines for data type handling
+* parse_utilcmd.c parse analysis for utility commands (done at execution time)
See also src/common/keywords.c, which contains the table of standard
keywords and the keyword lookup function. We separated that out because
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI
index 48d6e025d83..be9a6f96cfe 100644
--- a/src/backend/storage/lmgr/README-SSI
+++ b/src/backend/storage/lmgr/README-SSI
@@ -414,14 +414,14 @@ to be added from scratch.
2. The existing in-memory lock structures were not suitable for
tracking SIREAD locks.
-- In PostgreSQL, tuple level locks are not held in RAM for
+ - In PostgreSQL, tuple level locks are not held in RAM for
any length of time; lock information is written to the tuples
involved in the transactions.
-- In PostgreSQL, existing lock structures have pointers to
+ - In PostgreSQL, existing lock structures have pointers to
memory which is related to a session. SIREAD locks need to persist
past the end of the originating transaction and even the session
which ran it.
-- PostgreSQL needs to be able to tolerate a large number of
+ - PostgreSQL needs to be able to tolerate a large number of
transactions executing while one long-running transaction stays open
-- the in-RAM techniques discussed in the papers wouldn't support
that.
diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README
index ee14362b8e1..3edfbe933db 100644
--- a/src/backend/utils/fmgr/README
+++ b/src/backend/utils/fmgr/README
@@ -171,7 +171,6 @@ For float4, float8, and int8, the PG_GETARG macros will hide whether the
types are pass-by-value or pass-by-reference. For example, if float8 is
pass-by-reference then PG_GETARG_FLOAT8 expands to
-
(* (float8 *) DatumGetPointer(fcinfo->args[number].value))
and would typically be called like this:
diff --git a/src/backend/utils/misc/README b/src/backend/utils/misc/README
index abfa4737577..eaedfbc6df3 100644
--- a/src/backend/utils/misc/README
+++ b/src/backend/utils/misc/README
@@ -35,21 +35,21 @@ OK or false if not. The function can optionally do a few other things:
additional information to the generic "invalid value for parameter FOO"
complaint that guc.c will emit. To do that, call
- void GUC_check_errdetail(const char *format, ...)
+ void GUC_check_errdetail(const char *format, ...)
where the format string and additional arguments follow the rules for
errdetail() arguments. The resulting string will be emitted as the
DETAIL line of guc.c's error report, so it should follow the message style
guidelines for DETAIL messages. There is also
- void GUC_check_errhint(const char *format, ...)
+ void GUC_check_errhint(const char *format, ...)
which can be used in the same way to append a HINT message.
Occasionally it may even be appropriate to override guc.c's generic primary
message or error code, which can be done with
- void GUC_check_errcode(int sqlerrcode)
- void GUC_check_errmsg(const char *format, ...)
+ void GUC_check_errcode(int sqlerrcode)
+ void GUC_check_errmsg(const char *format, ...)
In general, check_hooks should avoid throwing errors directly if possible,
though this may be impractical to avoid for some corner cases such as
diff --git a/src/interfaces/ecpg/preproc/README.parser b/src/interfaces/ecpg/preproc/README.parser
index ec517a3b5a3..b647e248aba 100644
--- a/src/interfaces/ecpg/preproc/README.parser
+++ b/src/interfaces/ecpg/preproc/README.parser
@@ -25,24 +25,24 @@ rules concatenated together. e.g. if gram.y has this:
then "dumpedtokens" is "ruleAtokenAtokenBtokenC".
"postfix" above can be:
-a) "block" - the automatic rule created by parse.pl is completely
+1) "block" - the automatic rule created by parse.pl is completely
overridden, the code block has to be written completely as
it were in a plain bison grammar
-b) "rule" - the automatic rule is extended on, so new syntaxes
+2) "rule" - the automatic rule is extended on, so new syntaxes
are accepted for "ruleA". E.g.:
- ECPG: ruleAtokenAtokenBtokenC rule
- | tokenD tokenE { action_code; }
- ...
+ ECPG: ruleAtokenAtokenBtokenC rule
+ | tokenD tokenE { action_code; }
+ ...
It will be substituted with:
- ruleA: <original syntax forms and actions up to and including
- "tokenA tokenB tokenC">
- | tokenD tokenE { action_code; }
- ...
+ ruleA: <original syntax forms and actions up to and including
+ "tokenA tokenB tokenC">
+ | tokenD tokenE { action_code; }
+ ...
-c) "addon" - the automatic action for the rule (SQL syntax constructed
+3) "addon" - the automatic action for the rule (SQL syntax constructed
from the tokens concatenated together) is prepended with a new
action code part. This code part is written as is's already inside
the { ... }
diff --git a/src/port/README b/src/port/README
index e5aeed07b61..6a8594aba8d 100644
--- a/src/port/README
+++ b/src/port/README
@@ -14,15 +14,15 @@ libraries. This is done by removing -lpgport from the link line:
# Need to recompile any libpgport object files
LIBS := $(filter-out -lpgport, $(LIBS))
- and adding infrastructure to recompile the object files:
+ and adding infrastructure to recompile the object files:
OBJS= execute.o typename.o descriptor.o data.o error.o prepare.o memory.o \
connect.o misc.o path.o exec.o \
$(filter strlcat.o, $(LIBOBJS))
-The problem is that there is no testing of which object files need to be
-added, but missing functions usually show up when linking user
-applications.
+ The problem is that there is no testing of which object files need to be
+ added, but missing functions usually show up when linking user
+ applications.
2) For applications, we use -lpgport before -lpq, so the static files
from libpgport are linked first. This avoids having applications
diff --git a/src/test/recovery/README b/src/test/recovery/README
index c18fe7e21e9..d13156a140d 100644
--- a/src/test/recovery/README
+++ b/src/test/recovery/README
@@ -15,11 +15,11 @@ in addition to the core code.
Run
- make check
+ make check
or
- make installcheck
+ make installcheck
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
diff --git a/src/test/ssl/README b/src/test/ssl/README
index 957694c6892..c29f636c42e 100644
--- a/src/test/ssl/README
+++ b/src/test/ssl/README
@@ -43,19 +43,19 @@ Certificates
The test suite needs a set of public/private key pairs and certificates to
run:
-* root_ca
+* root_ca:
root CA, use to sign the server and client CA certificates.
-* server_ca
+* server_ca:
CA used to sign server certificates.
-* client_ca
+* client_ca:
CA used to sign client certificates.
-* server-cn-only
-server-cn-and-alt-names
-server-single-alt-name
-server-multiple-alt-names
+* server-cn-only,
+server-cn-and-alt-names,
+server-single-alt-name,
+server-multiple-alt-names,
server-no-names:
server certificates, with small variations in the hostnames present
in the certificate. Signed by server_ca.
diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README
index d984af8f1db..ac533481c01 100644
--- a/src/tools/pgindent/README
+++ b/src/tools/pgindent/README
@@ -23,11 +23,11 @@ PREREQUISITES:
("man perlmodinstall" explains it). Or, if you have cpan installed,
this should work:
- cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+ cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
Or if you have cpanm installed, you can just use:
- cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
+ cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
@@ -36,7 +36,7 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
2) Run pgindent on the C files:
- src/tools/pgindent/pgindent .
+ src/tools/pgindent/pgindent .
If any files generate errors, restore their original versions with
"git checkout", and see below for cleanup ideas.
@@ -55,9 +55,9 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
6) Do a full test build:
- make -s clean
- make -s all # look for unexpected warnings, and errors of course
- make check-world
+ make -s clean
+ make -s all # look for unexpected warnings, and errors of course
+ make check-world
Your configure switches should include at least --enable-tap-tests
or else much of the Perl code won't get exercised.
@@ -70,7 +70,7 @@ AT LEAST ONCE PER RELEASE CYCLE:
--------------------------------
1) Download the latest typedef file from the buildfarm:
- wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
+ wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
This step resolves any differences between the incrementally updated
version of the file and a clean, autogenerated one.
@@ -81,17 +81,17 @@ AT LEAST ONCE PER RELEASE CYCLE:
3) Indent the Perl code using perltidy:
- src/tools/pgindent/pgperltidy .
+ src/tools/pgindent/pgperltidy .
If you want to use some perltidy version that's not in your PATH,
first set the PERLTIDY environment variable to point to it.
4) Reformat the bootstrap catalog data files:
- ./configure # "make" will not work in an unconfigured tree
- cd src/include/catalog
- make reformat-dat-files
- cd ../../..
+ ./configure # "make" will not work in an unconfigured tree
+ cd src/include/catalog
+ make reformat-dat-files
+ cd ../../..
5) When you're done, "git commit" everything including the typedefs.list file
you used.
--
2.46.0
Attachments:
[text/plain] 0001-fixup-Standardize-syntax-in-internal-documentation.patch (18.6K, ../../[email protected]/2-0001-fixup-Standardize-syntax-in-internal-documentation.patch)
download | inline diff:
From 990329d7a8e6110ecf099162680f6c0eaf38ea14 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Mon, 23 Sep 2024 13:13:18 +0200
Subject: [PATCH] fixup! Standardize syntax in internal documentation
---
contrib/start-scripts/macos/README | 6 ++--
src/backend/access/gin/README | 6 ++--
src/backend/access/gist/README | 8 ++---
src/backend/access/hash/README | 2 +-
src/backend/lib/README | 22 ++++++------
src/backend/optimizer/plan/README | 6 ++--
src/backend/parser/README | 42 +++++++++++------------
src/backend/storage/lmgr/README-SSI | 6 ++--
src/backend/utils/fmgr/README | 1 -
src/backend/utils/misc/README | 8 ++---
src/interfaces/ecpg/preproc/README.parser | 20 +++++------
src/port/README | 8 ++---
src/test/recovery/README | 4 +--
src/test/ssl/README | 14 ++++----
src/tools/pgindent/README | 24 ++++++-------
15 files changed, 88 insertions(+), 89 deletions(-)
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README
index 8fe6efb657d..c71f98880ca 100644
--- a/contrib/start-scripts/macos/README
+++ b/contrib/start-scripts/macos/README
@@ -15,11 +15,11 @@ if you plan to run the Postgres server under some user name other
than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
- /Library/LaunchDaemons/. You must do this as root:
+/Library/LaunchDaemons/. You must do this as root:
- sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
+ sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
- because the file will be ignored if it is not root-owned.
+ because the file will be ignored if it is not root-owned.
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README
index 08faa944105..ccff6896513 100644
--- a/src/backend/access/gin/README
+++ b/src/backend/access/gin/README
@@ -124,9 +124,9 @@ know are:
or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
the "category" of the null entry. These are the possible categories:
- 1 = ordinary null key value extracted from an indexable item
- 2 = placeholder for zero-key indexable item
- 3 = placeholder for null indexable item
+ 1 = ordinary null key value extracted from an indexable item
+ 2 = placeholder for zero-key indexable item
+ 3 = placeholder for null indexable item
Placeholder null entries are inserted into the index because otherwise
there would be no index entry at all for an empty or null indexable item,
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README
index af082fc2bb0..62818b09d79 100644
--- a/src/backend/access/gist/README
+++ b/src/backend/access/gist/README
@@ -477,7 +477,7 @@ value. The page is not recycled, until that XID is no longer visible to
anyone. That's much more conservative than necessary, but let's keep it
simple.
-
-Authors:
- Teodor Sigaev <[email protected]>
- Oleg Bartunov <[email protected]>
+Authors
+-------
+* Teodor Sigaev <[email protected]>
+* Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README
index a5df13a68a2..5b4c758df62 100644
--- a/src/backend/access/hash/README
+++ b/src/backend/access/hash/README
@@ -255,7 +255,7 @@ The reader algorithm is:
release the buffer content lock on old bucket, but not pin
retake the buffer content lock on new bucket
arrange to scan the old bucket normally and the new bucket for
- tuples which are not moved-by-split
+ tuples which are not moved-by-split
-- then, per read request:
reacquire content lock on current page
step to next page if necessary (no chaining of content locks, but keep
diff --git a/src/backend/lib/README b/src/backend/lib/README
index fc8e1aa1f7c..735e3790513 100644
--- a/src/backend/lib/README
+++ b/src/backend/lib/README
@@ -1,27 +1,27 @@
This directory contains a general purpose data structures, for use anywhere
in the backend:
- binaryheap.c - a binary heap
+* binaryheap.c - a binary heap
- bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
+* bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
- bloomfilter.c - probabilistic, space-efficient set membership testing
+* bloomfilter.c - probabilistic, space-efficient set membership testing
- dshash.c - concurrent hash tables backed by dynamic shared memory areas
+* dshash.c - concurrent hash tables backed by dynamic shared memory areas
- hyperloglog.c - a streaming cardinality estimator
+* hyperloglog.c - a streaming cardinality estimator
- ilist.c - single and double-linked lists
+* ilist.c - single and double-linked lists
- integerset.c - a data structure for holding large set of integers
+* integerset.c - a data structure for holding large set of integers
- knapsack.c - knapsack problem solver
+* knapsack.c - knapsack problem solver
- pairingheap.c - a pairing heap
+* pairingheap.c - a pairing heap
- rbtree.c - a red-black tree
+* rbtree.c - a red-black tree
- stringinfo.c - an extensible string type
+* stringinfo.c - an extensible string type
Aside from the inherent characteristics of the data structures, there are a
diff --git a/src/backend/optimizer/plan/README b/src/backend/optimizer/plan/README
index 93dd422dc15..04d9c50c51e 100644
--- a/src/backend/optimizer/plan/README
+++ b/src/backend/optimizer/plan/README
@@ -91,9 +91,9 @@ on each call) ExecReScan() now supports most of Plan types...
Explanation of EXPLAIN.
-vac=> explain select * from tmp where x >= (select max(x2) from test2
-where y2 = y and exists (select * from tempx where tx = x));
-NOTICE: QUERY PLAN:
+ vac=> explain select * from tmp where x >= (select max(x2) from test2
+ where y2 = y and exists (select * from tempx where tx = x));
+ NOTICE: QUERY PLAN:
Seq Scan on tmp (cost=40.03 size=101 width=8)
SubPlan
diff --git a/src/backend/parser/README b/src/backend/parser/README
index e6016fa4301..9ac8026c73a 100644
--- a/src/backend/parser/README
+++ b/src/backend/parser/README
@@ -7,27 +7,27 @@ This directory does more than tokenize and parse SQL queries. It also
creates Query structures for the various complex queries that are passed
to the optimizer and then executor.
- parser.c things start here
- scan.l break query into tokens
- scansup.c handle escapes in input strings
- gram.y parse the tokens and produce a "raw" parse tree
- analyze.c top level of parse analysis for optimizable queries
- parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
- parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
- parse_coerce.c handle coercing expressions to different data types
- parse_collate.c assign collation information in completed expressions
- parse_cte.c handle Common Table Expressions (WITH clauses)
- parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
- parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
- parse_func.c handle functions, table.column and column identifiers
- parse_merge.c handle MERGE
- parse_node.c create nodes for various structures
- parse_oper.c handle operators in expressions
- parse_param.c handle Params (for the cases used in the core backend)
- parse_relation.c support routines for tables and column handling
- parse_target.c handle the result list of the query
- parse_type.c support routines for data type handling
- parse_utilcmd.c parse analysis for utility commands (done at execution time)
+* parser.c things start here
+* scan.l break query into tokens
+* scansup.c handle escapes in input strings
+* gram.y parse the tokens and produce a "raw" parse tree
+* analyze.c top level of parse analysis for optimizable queries
+* parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
+* parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
+* parse_coerce.c handle coercing expressions to different data types
+* parse_collate.c assign collation information in completed expressions
+* parse_cte.c handle Common Table Expressions (WITH clauses)
+* parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
+* parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
+* parse_func.c handle functions, table.column and column identifiers
+* parse_merge.c handle MERGE
+* parse_node.c create nodes for various structures
+* parse_oper.c handle operators in expressions
+* parse_param.c handle Params (for the cases used in the core backend)
+* parse_relation.c support routines for tables and column handling
+* parse_target.c handle the result list of the query
+* parse_type.c support routines for data type handling
+* parse_utilcmd.c parse analysis for utility commands (done at execution time)
See also src/common/keywords.c, which contains the table of standard
keywords and the keyword lookup function. We separated that out because
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI
index 48d6e025d83..be9a6f96cfe 100644
--- a/src/backend/storage/lmgr/README-SSI
+++ b/src/backend/storage/lmgr/README-SSI
@@ -414,14 +414,14 @@ to be added from scratch.
2. The existing in-memory lock structures were not suitable for
tracking SIREAD locks.
-- In PostgreSQL, tuple level locks are not held in RAM for
+ - In PostgreSQL, tuple level locks are not held in RAM for
any length of time; lock information is written to the tuples
involved in the transactions.
-- In PostgreSQL, existing lock structures have pointers to
+ - In PostgreSQL, existing lock structures have pointers to
memory which is related to a session. SIREAD locks need to persist
past the end of the originating transaction and even the session
which ran it.
-- PostgreSQL needs to be able to tolerate a large number of
+ - PostgreSQL needs to be able to tolerate a large number of
transactions executing while one long-running transaction stays open
-- the in-RAM techniques discussed in the papers wouldn't support
that.
diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README
index ee14362b8e1..3edfbe933db 100644
--- a/src/backend/utils/fmgr/README
+++ b/src/backend/utils/fmgr/README
@@ -171,7 +171,6 @@ For float4, float8, and int8, the PG_GETARG macros will hide whether the
types are pass-by-value or pass-by-reference. For example, if float8 is
pass-by-reference then PG_GETARG_FLOAT8 expands to
-
(* (float8 *) DatumGetPointer(fcinfo->args[number].value))
and would typically be called like this:
diff --git a/src/backend/utils/misc/README b/src/backend/utils/misc/README
index abfa4737577..eaedfbc6df3 100644
--- a/src/backend/utils/misc/README
+++ b/src/backend/utils/misc/README
@@ -35,21 +35,21 @@ OK or false if not. The function can optionally do a few other things:
additional information to the generic "invalid value for parameter FOO"
complaint that guc.c will emit. To do that, call
- void GUC_check_errdetail(const char *format, ...)
+ void GUC_check_errdetail(const char *format, ...)
where the format string and additional arguments follow the rules for
errdetail() arguments. The resulting string will be emitted as the
DETAIL line of guc.c's error report, so it should follow the message style
guidelines for DETAIL messages. There is also
- void GUC_check_errhint(const char *format, ...)
+ void GUC_check_errhint(const char *format, ...)
which can be used in the same way to append a HINT message.
Occasionally it may even be appropriate to override guc.c's generic primary
message or error code, which can be done with
- void GUC_check_errcode(int sqlerrcode)
- void GUC_check_errmsg(const char *format, ...)
+ void GUC_check_errcode(int sqlerrcode)
+ void GUC_check_errmsg(const char *format, ...)
In general, check_hooks should avoid throwing errors directly if possible,
though this may be impractical to avoid for some corner cases such as
diff --git a/src/interfaces/ecpg/preproc/README.parser b/src/interfaces/ecpg/preproc/README.parser
index ec517a3b5a3..b647e248aba 100644
--- a/src/interfaces/ecpg/preproc/README.parser
+++ b/src/interfaces/ecpg/preproc/README.parser
@@ -25,24 +25,24 @@ rules concatenated together. e.g. if gram.y has this:
then "dumpedtokens" is "ruleAtokenAtokenBtokenC".
"postfix" above can be:
-a) "block" - the automatic rule created by parse.pl is completely
+1) "block" - the automatic rule created by parse.pl is completely
overridden, the code block has to be written completely as
it were in a plain bison grammar
-b) "rule" - the automatic rule is extended on, so new syntaxes
+2) "rule" - the automatic rule is extended on, so new syntaxes
are accepted for "ruleA". E.g.:
- ECPG: ruleAtokenAtokenBtokenC rule
- | tokenD tokenE { action_code; }
- ...
+ ECPG: ruleAtokenAtokenBtokenC rule
+ | tokenD tokenE { action_code; }
+ ...
It will be substituted with:
- ruleA: <original syntax forms and actions up to and including
- "tokenA tokenB tokenC">
- | tokenD tokenE { action_code; }
- ...
+ ruleA: <original syntax forms and actions up to and including
+ "tokenA tokenB tokenC">
+ | tokenD tokenE { action_code; }
+ ...
-c) "addon" - the automatic action for the rule (SQL syntax constructed
+3) "addon" - the automatic action for the rule (SQL syntax constructed
from the tokens concatenated together) is prepended with a new
action code part. This code part is written as is's already inside
the { ... }
diff --git a/src/port/README b/src/port/README
index e5aeed07b61..6a8594aba8d 100644
--- a/src/port/README
+++ b/src/port/README
@@ -14,15 +14,15 @@ libraries. This is done by removing -lpgport from the link line:
# Need to recompile any libpgport object files
LIBS := $(filter-out -lpgport, $(LIBS))
- and adding infrastructure to recompile the object files:
+ and adding infrastructure to recompile the object files:
OBJS= execute.o typename.o descriptor.o data.o error.o prepare.o memory.o \
connect.o misc.o path.o exec.o \
$(filter strlcat.o, $(LIBOBJS))
-The problem is that there is no testing of which object files need to be
-added, but missing functions usually show up when linking user
-applications.
+ The problem is that there is no testing of which object files need to be
+ added, but missing functions usually show up when linking user
+ applications.
2) For applications, we use -lpgport before -lpq, so the static files
from libpgport are linked first. This avoids having applications
diff --git a/src/test/recovery/README b/src/test/recovery/README
index c18fe7e21e9..d13156a140d 100644
--- a/src/test/recovery/README
+++ b/src/test/recovery/README
@@ -15,11 +15,11 @@ in addition to the core code.
Run
- make check
+ make check
or
- make installcheck
+ make installcheck
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
diff --git a/src/test/ssl/README b/src/test/ssl/README
index 957694c6892..c29f636c42e 100644
--- a/src/test/ssl/README
+++ b/src/test/ssl/README
@@ -43,19 +43,19 @@ Certificates
The test suite needs a set of public/private key pairs and certificates to
run:
-* root_ca
+* root_ca:
root CA, use to sign the server and client CA certificates.
-* server_ca
+* server_ca:
CA used to sign server certificates.
-* client_ca
+* client_ca:
CA used to sign client certificates.
-* server-cn-only
-server-cn-and-alt-names
-server-single-alt-name
-server-multiple-alt-names
+* server-cn-only,
+server-cn-and-alt-names,
+server-single-alt-name,
+server-multiple-alt-names,
server-no-names:
server certificates, with small variations in the hostnames present
in the certificate. Signed by server_ca.
diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README
index d984af8f1db..ac533481c01 100644
--- a/src/tools/pgindent/README
+++ b/src/tools/pgindent/README
@@ -23,11 +23,11 @@ PREREQUISITES:
("man perlmodinstall" explains it). Or, if you have cpan installed,
this should work:
- cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+ cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
Or if you have cpanm installed, you can just use:
- cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
+ cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
@@ -36,7 +36,7 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
2) Run pgindent on the C files:
- src/tools/pgindent/pgindent .
+ src/tools/pgindent/pgindent .
If any files generate errors, restore their original versions with
"git checkout", and see below for cleanup ideas.
@@ -55,9 +55,9 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
6) Do a full test build:
- make -s clean
- make -s all # look for unexpected warnings, and errors of course
- make check-world
+ make -s clean
+ make -s all # look for unexpected warnings, and errors of course
+ make check-world
Your configure switches should include at least --enable-tap-tests
or else much of the Perl code won't get exercised.
@@ -70,7 +70,7 @@ AT LEAST ONCE PER RELEASE CYCLE:
--------------------------------
1) Download the latest typedef file from the buildfarm:
- wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
+ wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
This step resolves any differences between the incrementally updated
version of the file and a clean, autogenerated one.
@@ -81,17 +81,17 @@ AT LEAST ONCE PER RELEASE CYCLE:
3) Indent the Perl code using perltidy:
- src/tools/pgindent/pgperltidy .
+ src/tools/pgindent/pgperltidy .
If you want to use some perltidy version that's not in your PATH,
first set the PERLTIDY environment variable to point to it.
4) Reformat the bootstrap catalog data files:
- ./configure # "make" will not work in an unconfigured tree
- cd src/include/catalog
- make reformat-dat-files
- cd ../../..
+ ./configure # "make" will not work in an unconfigured tree
+ cd src/include/catalog
+ make reformat-dat-files
+ cd ../../..
5) When you're done, "git commit" everything including the typedefs.list file
you used.
--
2.46.0
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-10-01 13:51 Daniel Gustafsson <[email protected]>
parent: Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Daniel Gustafsson @ 2024-10-01 13:51 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
> On 23 Sep 2024, at 13:58, Peter Eisentraut <[email protected]> wrote:
> I went through this file by file and checked the results of a markdown-to-HTML conversion using cmark and looking at the raw output source files.
Thanks for reviewing!
I thought the consensus of the thread was to skip Markdown compatibility and
only go for more consistency in the current format, so I didn't check any such
results when proposing that version. I've checked with the Markdown rendering
preview in VSCode for this.
Placing the goalposts in the right place seems useful, should we aim for
Markdown, or a consistent look-and-feel regardless of markdown compatibility.
> A lot of the changes are obvious and make sense. But there are a number of cases of code within lists or nested lists or both that need further careful investigation. I'm attaching a fixup patch where I tried to improve some of this (and a few other things I found along the way). Some of the more complicated ones, such as src/backend/storage/lmgr/README-SSI, will need to be checked again and even more carefully to make sure that the meaning is not altered by these patches.
Agreed. Checking it I don't see any such cases but more careful looking is
needed.
> One underlying problem that I see is that markdown assumes four-space tabs, but a standard editor configuration (and apparently your editor) uses 8 tabs. But then, if you have a common situation like
>
> ```
> 1. Run this code
>
> <tab>$ sudo kill
> ```
>
> then that's incorrect (the code line will not be inside the list), because it should be
>
> ```
> 1. Run this code
>
> <tab><tab>$ sudo kill
> ```
>
> or
>
> ```
> 1. Run this code
>
> <8 spaces>$ sudo kill
> ```
>
> So we need to think about a way to make this more robust for future people editing. Maybe something in .gitattributes or some editor settings. Otherwise, it will be all over the places after a while.
Maybe we can add some form of pandoc target for rendering as as way to test
locally before pushing? (For those with pandoc installed, but we already have
the infrastructure in meson to use pandoc so it could be convenient perhaps).
> (There are also a couple of places where apparently you changed whitespace that wasn't necessary to be changed.)
It was to make the files look and feel consistent, I've tried to reduce that in
the attached.
> Apart from this, I don't changing the placeholders like <foo> to < foo >. In some cases, this really decreases readability. Maybe we should look for different approaches there.
Agreed. I took a stab at some of them in the attached. The usage in
src/test/isolation/README is seemingly the hardest to replace and I'm not sure
how we should proceed there.
> Maybe there are some easy changes that could be extracted from this patch, but the whitespace and list issue needs more consideration.
If we want to reduce the size of this I think the changes which add a line of
whitespace could be broken out and committed separately. One random example
from the diff being:
***
Outer join identity 3 (discussed above) complicates this picture
a bit. In the form
+
A leftjoin (B leftjoin C on (Pbc)) on (Pab)
+
all of the Vars in clauses Pbc and Pab will have empty varnullingrels,
but if we start with
***
Personally I consider those changes a win for readability on their own
regardless of any progress towards Markdown.
The attached has your changed rolled into 0001 and any new changes in 0002 for
ease of skimming the diffs.
--
Daniel Gustafsson
Attachments:
[application/octet-stream] v4-0002-Review-comments.patch (35.6K, ../../[email protected]/2-v4-0002-Review-comments.patch)
download | inline diff:
From 0fd4d7e5ead4ed7491015dee87bcd22e91ce93f4 Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <[email protected]>
Date: Tue, 1 Oct 2024 15:35:37 +0200
Subject: [PATCH v4 2/2] Review comments
---
contrib/start-scripts/macos/README | 5 +-
src/backend/access/gin/README | 110 ++++++-------
src/backend/access/spgist/README | 2 +-
src/backend/access/transam/README | 4 +-
src/backend/regex/README | 46 +++---
src/backend/storage/lmgr/README-SSI | 240 ++++++++++++++--------------
src/backend/utils/mb/README | 16 +-
src/tools/ci/README | 6 +-
8 files changed, 212 insertions(+), 217 deletions(-)
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README
index c71f98880c..5382c2e45f 100644
--- a/contrib/start-scripts/macos/README
+++ b/contrib/start-scripts/macos/README
@@ -15,12 +15,11 @@ if you plan to run the Postgres server under some user name other
than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
-/Library/LaunchDaemons/. You must do this as root:
+/Library/LaunchDaemons/. You must do this as root because the file
+will be ignored if it is not root-owned:
sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
- because the file will be ignored if it is not root-owned.
-
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README
index ccff689651..76f9b68fef 100644
--- a/src/backend/access/gin/README
+++ b/src/backend/access/gin/README
@@ -111,81 +111,77 @@ by building a "normal" index tuple and then modifying it.) The points to
know are:
* In a single-column index, a key tuple just contains the key datum, but
- in a multi-column index, a key tuple contains the pair (column number,
- key datum) where the column number is stored as an int2. This is needed
- to support different key data types in different columns. This much of
- the tuple is built by index_form_tuple according to the usual rules.
- The column number (if present) can never be null, but the key datum can
- be, in which case a null bitmap is present as usual. (As usual for index
- tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
+in a multi-column index, a key tuple contains the pair (column number,
+key datum) where the column number is stored as an int2. This is needed
+to support different key data types in different columns. This much of
+the tuple is built by index_form_tuple according to the usual rules.
+The column number (if present) can never be null, but the key datum can
+be, in which case a null bitmap is present as usual. (As usual for index
+tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
* If the key datum is null (ie, IndexTupleHasNulls() is true), then
- just after the nominal index data (ie, at offset IndexInfoFindDataOffset
- or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
- the "category" of the null entry. These are the possible categories:
-
- 1 = ordinary null key value extracted from an indexable item
- 2 = placeholder for zero-key indexable item
- 3 = placeholder for null indexable item
-
- Placeholder null entries are inserted into the index because otherwise
- there would be no index entry at all for an empty or null indexable item,
- which would mean that full index scans couldn't be done and various corner
- cases would give wrong answers. The different categories of null entries
- are treated as distinct keys by the btree, but heap itempointers for the
- same category of null entry are merged into one index entry just as happens
- with ordinary key entries.
+just after the nominal index data (ie, at offset IndexInfoFindDataOffset
+or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
+the "category" of the null entry. These are the possible categories:
+ 1) ordinary null key value extracted from an indexable item
+ 2) placeholder for zero-key indexable item
+ 3) placeholder for null indexable item
+Placeholder null entries are inserted into the index because otherwise
+there would be no index entry at all for an empty or null indexable item,
+which would mean that full index scans couldn't be done and various corner
+cases would give wrong answers. The different categories of null entries
+are treated as distinct keys by the btree, but heap itempointers for the
+same category of null entry are merged into one index entry just as happens
+with ordinary key entries.
* In a key entry at the btree leaf level, at the next SHORTALIGN boundary,
- there is a list of item pointers, in compressed format (see Posting List
- Compression section), pointing to the heap tuples for which the indexable
- items contain this key. This is called the "posting list".
-
- If the list would be too big for the index tuple to fit on an index page, the
- ItemPointers are pushed out to a separate posting page or pages, and none
- appear in the key entry itself. The separate pages are called a "posting
- tree" (see below); Note that in either case, the ItemPointers associated with
- a key can easily be read out in sorted order; this is relied on by the scan
- algorithms.
+there is a list of item pointers, in compressed format (see Posting List
+Compression section), pointing to the heap tuples for which the indexable
+items contain this key. This is called the "posting list".
+If the list would be too big for the index tuple to fit on an index page, the
+ItemPointers are pushed out to a separate posting page or pages, and none
+appear in the key entry itself. The separate pages are called a "posting
+tree" (see below); Note that in either case, the ItemPointers associated with
+a key can easily be read out in sorted order; this is relied on by the scan
+algorithms.
* The index tuple header fields of a leaf key entry are abused as follows:
+ 1) Posting list case:
-1) Posting list case:
-
-* ItemPointerGetBlockNumber(&itup->t_tid) contains the offset from index
+ - ItemPointerGetBlockNumber(&itup->t_tid) contains the offset from index
tuple start to the posting list.
Access macros: GinGetPostingOffset(itup) / GinSetPostingOffset(itup,n)
-* ItemPointerGetOffsetNumber(&itup->t_tid) contains the number of elements
+ - ItemPointerGetOffsetNumber(&itup->t_tid) contains the number of elements
in the posting list (number of heap itempointers).
Access macros: GinGetNPosting(itup) / GinSetNPosting(itup,n)
-* If IndexTupleHasNulls(itup) is true, the null category byte can be
+ - If IndexTupleHasNulls(itup) is true, the null category byte can be
accessed/set with GinGetNullCategory(itup,gs) / GinSetNullCategory(itup,gs,c)
-* The posting list can be accessed with GinGetPosting(itup)
+ - The posting list can be accessed with GinGetPosting(itup)
-* If GinItupIsCompressed(itup), the posting list is stored in compressed
+ - If GinItupIsCompressed(itup), the posting list is stored in compressed
format. Otherwise it is just an array of ItemPointers. New tuples are always
stored in compressed format, uncompressed items can be present if the
database was migrated from 9.3 or earlier version.
-2) Posting tree case:
+ 2) Posting tree case:
-* ItemPointerGetBlockNumber(&itup->t_tid) contains the index block number
+ - ItemPointerGetBlockNumber(&itup->t_tid) contains the index block number
of the root of the posting tree.
Access macros: GinGetPostingTree(itup) / GinSetPostingTree(itup, blkno)
-* ItemPointerGetOffsetNumber(&itup->t_tid) contains the magic number
+ - ItemPointerGetOffsetNumber(&itup->t_tid) contains the magic number
GIN_TREE_POSTING, which distinguishes this from the posting-list case
(it's large enough that that many heap itempointers couldn't possibly
fit on an index page). This value is inserted automatically by the
GinSetPostingTree macro.
-* If IndexTupleHasNulls(itup) is true, the null category byte can be
+ - If IndexTupleHasNulls(itup) is true, the null category byte can be
accessed/set with GinGetNullCategory(itup,gs) / GinSetNullCategory(itup,gs,c)
-* The posting list is not present and must not be accessed.
+ - The posting list is not present and must not be accessed.
Use the macro GinIsPostingTree(itup) to determine which case applies.
@@ -518,44 +514,44 @@ For compatibility, old uncompressed format is also supported. Following
rules are used to handle it:
* GIN_ITUP_COMPRESSED flag marks index tuples that contain a posting list.
- This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
- Use GinItupIsCompressed(itup) to check the flag.
+This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
+Use GinItupIsCompressed(itup) to check the flag.
* Posting tree pages in the new format are marked with the GIN_COMPRESSED flag.
Macros GinPageIsCompressed(page) and GinPageSetCompressed(page) are used to
check and set this flag.
* All scan operations check format of posting list add use corresponding code
- to read its content.
+to read its content.
* When updating an index tuple containing an uncompressed posting list, it
- will be replaced with new index tuple containing a compressed list.
+will be replaced with new index tuple containing a compressed list.
* When updating an uncompressed posting tree leaf page, it's compressed.
* If vacuum finds some dead TIDs in uncompressed posting lists, they are
- converted into compressed posting lists. This assumes that the compressed
- posting list fits in the space occupied by the uncompressed list. IOW, we
- assume that the compressed version of the page, with the dead items removed,
- takes less space than the old uncompressed version.
+converted into compressed posting lists. This assumes that the compressed
+posting list fits in the space occupied by the uncompressed list. IOW, we
+assume that the compressed version of the page, with the dead items removed,
+takes less space than the old uncompressed version.
Limitations
-----------
-* Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
-* Gin searches entries only by equality matching, or simple range
- matching using the "partial match" feature.
+ * Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
+ * Gin searches entries only by equality matching, or simple range
+ matching using the "partial match" feature.
TODO
----
Nearest future:
-* Opclasses for more types (no programming, just many catalog changes)
+ * Opclasses for more types (no programming, just many catalog changes)
Distant future:
-* Replace B-tree of entries to something like GiST
+ * Replace B-tree of entries to something like GiST
Authors
-------
diff --git a/src/backend/access/spgist/README b/src/backend/access/spgist/README
index 2b890e4f8e..b55bebf438 100644
--- a/src/backend/access/spgist/README
+++ b/src/backend/access/spgist/README
@@ -160,7 +160,7 @@ the following notation, where tuple's id is just for discussion (no such id
is actually stored):
inner tuple: {tuple id}(prefix string)[ comma separated list of node labels ]
-leaf tuple: {tuple id}< value >
+leaf tuple: {tuple id}(value)
Suppose we need to insert string 'www.gogo.com' into inner tuple
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README
index 62015da1fe..7adab8a06b 100644
--- a/src/backend/access/transam/README
+++ b/src/backend/access/transam/README
@@ -90,7 +90,7 @@ commit or abort processing.
Furthermore, suppose the "SELECT * FROM foo" caused an abort condition. In
this case AbortCurrentTransaction is called, and the transaction is put in
aborted state. In this state, any user input is ignored except for
-transaction-termination statements, or ROLLBACK TO <savepoint> commands.
+transaction-termination statements, or ROLLBACK TO SAVEPOINT commands.
Transaction aborts can occur in two ways:
@@ -171,7 +171,7 @@ CommitTransactionCommand, the real work is done. The main point of doing
things this way is that if we get an error while popping state stack entries,
the remaining stack entries still show what we need to do to finish up.
-In the case of ROLLBACK TO < savepoint >, we abort all the subtransactions up
+In the case of ROLLBACK TO SAVEPOINT, we abort all the subtransactions up
through the one identified by the savepoint name, and then re-create that
subtransaction level with the same name. So it's a completely new
subtransaction as far as the internals are concerned.
diff --git a/src/backend/regex/README b/src/backend/regex/README
index a0d45589b6..d8528b2af8 100644
--- a/src/backend/regex/README
+++ b/src/backend/regex/README
@@ -10,11 +10,11 @@ General source-file layout
There are six separately-compilable source files, five of which expose
exactly one exported function apiece:
- regcomp.c pg_regcomp
- regexec.c pg_regexec
- regerror.c pg_regerror
- regfree.c pg_regfree
- regprefix.c pg_regprefix
+* regcomp.c pg_regcomp
+* regexec.c pg_regexec
+* regerror.c pg_regerror
+* regfree.c pg_regfree
+* regprefix.c pg_regprefix
(The pg_ prefixes were added by the Postgres project to distinguish this
library version from any similar one that might be present on a particular
@@ -40,19 +40,19 @@ structs.)
What's where in src/backend/regex/:
- regcomp.c Top-level regex compilation code
- regc_color.c Color map management
- regc_cvec.c Character vector (cvec) management
- regc_lex.c Lexer
- regc_nfa.c NFA handling
- regc_locale.c Application-specific locale code from Tcl project
- regc_pg_locale.c Postgres-added application-specific locale code
- regexec.c Top-level regex execution code
- rege_dfa.c DFA creation and execution
- regerror.c pg_regerror: generate text for a regex error code
- regfree.c pg_regfree: API to free a no-longer-needed regex_t
- regexport.c Functions for extracting info from a regex_t
- regprefix.c Code for extracting a common prefix from a regex_t
+* regcomp.c Top-level regex compilation code
+* regc_color.c Color map management
+* regc_cvec.c Character vector (cvec) management
+* regc_lex.c Lexer
+* regc_nfa.c NFA handling
+* regc_locale.c Application-specific locale code from Tcl project
+* regc_pg_locale.c Postgres-added application-specific locale code
+* regexec.c Top-level regex execution code
+* rege_dfa.c DFA creation and execution
+* regerror.c pg_regerror: generate text for a regex error code
+* regfree.c pg_regfree: API to free a no-longer-needed regex_t
+* regexport.c Functions for extracting info from a regex_t
+* regprefix.c Code for extracting a common prefix from a regex_t
The locale-specific code is concerned primarily with case-folding and with
expanding locale-specific character classes, such as [[:alnum:]]. It
@@ -60,11 +60,11 @@ really needs refactoring if this is ever to become a standalone library.
The header files for the library are in src/include/regex/:
- regcustom.h Customizes library for particular application
- regerrs.h Error message list
- regex.h Exported API
- regexport.h Exported API for regexport.c
- regguts.h Internals declarations
+* regcustom.h Customizes library for particular application
+* regerrs.h Error message list
+* regex.h Exported API
+* regexport.h Exported API for regexport.c
+* regguts.h Internals declarations
DFAs, NFAs, and all that
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI
index be9a6f96cf..016a48c408 100644
--- a/src/backend/storage/lmgr/README-SSI
+++ b/src/backend/storage/lmgr/README-SSI
@@ -200,54 +200,54 @@ PostgreSQL Implementation
-------------------------
* Since this technique is based on Snapshot Isolation (SI), those
- areas in PostgreSQL which don't use SI can't be brought under SSI.
- This includes system tables, temporary tables, sequences, hint bit
- rewrites, etc. SSI can not eliminate existing anomalies in these
- areas.
+areas in PostgreSQL which don't use SI can't be brought under SSI.
+This includes system tables, temporary tables, sequences, hint bit
+rewrites, etc. SSI can not eliminate existing anomalies in these
+areas.
* Any transaction which is run at a transaction isolation level
- other than SERIALIZABLE will not be affected by SSI. If you want to
- enforce business rules through SSI, all transactions should be run at
- the SERIALIZABLE transaction isolation level, and that should
- probably be set as the default.
+other than SERIALIZABLE will not be affected by SSI. If you want to
+enforce business rules through SSI, all transactions should be run at
+the SERIALIZABLE transaction isolation level, and that should
+probably be set as the default.
* If all transactions are run at the SERIALIZABLE transaction
- isolation level, business rules can be enforced in triggers or
- application code without ever having a need to acquire an explicit
- lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
+isolation level, business rules can be enforced in triggers or
+application code without ever having a need to acquire an explicit
+lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
* Those who want to continue to use snapshot isolation without
- the additional protections of SSI (and the associated costs of
- enforcing those protections), can use the REPEATABLE READ transaction
- isolation level. This level retains its legacy behavior, which
- is identical to the old SERIALIZABLE implementation and fully
- consistent with the standard's requirements for the REPEATABLE READ
- transaction isolation level.
+the additional protections of SSI (and the associated costs of
+enforcing those protections), can use the REPEATABLE READ transaction
+isolation level. This level retains its legacy behavior, which
+is identical to the old SERIALIZABLE implementation and fully
+consistent with the standard's requirements for the REPEATABLE READ
+transaction isolation level.
* Performance under this SSI implementation will be significantly
- improved if transactions which don't modify permanent tables are
- declared to be READ ONLY before they begin reading data.
+improved if transactions which don't modify permanent tables are
+declared to be READ ONLY before they begin reading data.
* Performance under SSI will tend to degrade more rapidly with a
- large number of active database transactions than under less strict
- isolation levels. Limiting the number of active transactions through
- use of a connection pool or similar techniques may be necessary to
- maintain good performance.
+large number of active database transactions than under less strict
+isolation levels. Limiting the number of active transactions through
+use of a connection pool or similar techniques may be necessary to
+maintain good performance.
* Any transaction which must be rolled back to prevent
- serialization anomalies will fail with SQLSTATE 40001, which has a
- standard meaning of "serialization failure".
+serialization anomalies will fail with SQLSTATE 40001, which has a
+standard meaning of "serialization failure".
* This SSI implementation makes an effort to choose the
- transaction to be canceled such that an immediate retry of the
- transaction will not fail due to conflicts with exactly the same
- transactions. Pursuant to this goal, no transaction is canceled
- until one of the other transactions in the set of conflicts which
- could generate an anomaly has successfully committed. This is
- conceptually similar to how write conflicts are handled. To fully
- implement this guarantee there needs to be a way to roll back the
- active transaction for another process with a serialization failure
- SQLSTATE, even if it is "idle in transaction".
+transaction to be canceled such that an immediate retry of the
+transaction will not fail due to conflicts with exactly the same
+transactions. Pursuant to this goal, no transaction is canceled
+until one of the other transactions in the set of conflicts which
+could generate an anomaly has successfully committed. This is
+conceptually similar to how write conflicts are handled. To fully
+implement this guarantee there needs to be a way to roll back the
+active transaction for another process with a serialization failure
+SQLSTATE, even if it is "idle in transaction".
Predicate Locking
@@ -306,20 +306,20 @@ Predicate locks will be acquired for the heap based on the following:
* For a table scan, the entire relation will be locked.
* Each tuple read which is visible to the reading transaction
- will be locked, whether or not it meets selection criteria; except
- that there is no need to acquire an SIREAD lock on a tuple when the
- transaction already holds a write lock on any tuple representing the
- row, since a rw-conflict would also create a ww-dependency which
- has more aggressive enforcement and thus will prevent any anomaly.
+will be locked, whether or not it meets selection criteria; except
+that there is no need to acquire an SIREAD lock on a tuple when the
+transaction already holds a write lock on any tuple representing the
+row, since a rw-conflict would also create a ww-dependency which
+has more aggressive enforcement and thus will prevent any anomaly.
* Modifying a heap tuple creates a rw-conflict with any transaction
- that holds a SIREAD lock on that tuple, or on the page or relation
- that contains it.
+that holds a SIREAD lock on that tuple, or on the page or relation
+that contains it.
* Inserting a new tuple creates a rw-conflict with any transaction
- holding a SIREAD lock on the entire relation. It doesn't conflict with
- page-level locks, because page-level locks are only used to aggregate
- tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
+holding a SIREAD lock on the entire relation. It doesn't conflict with
+page-level locks, because page-level locks are only used to aggregate
+tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
Index AM implementations
@@ -347,59 +347,59 @@ false positives, they should be minimized for performance reasons.
Several optimizations are possible, though not all are implemented yet:
* An index scan which is just finding the right position for an
- index insertion or deletion need not acquire a predicate lock.
+index insertion or deletion need not acquire a predicate lock.
* An index scan which is comparing for equality on the entire key
- for a unique index need not acquire a predicate lock as long as a key
- is found corresponding to a visible tuple which has not been modified
- by another transaction -- there are no "between or around" gaps to
- cover.
+for a unique index need not acquire a predicate lock as long as a key
+is found corresponding to a visible tuple which has not been modified
+by another transaction -- there are no "between or around" gaps to
+cover.
* As long as built-in foreign key enforcement continues to use
- its current "special tricks" to deal with MVCC issues, predicate
- locks should not be needed for scans done by enforcement code.
+its current "special tricks" to deal with MVCC issues, predicate
+locks should not be needed for scans done by enforcement code.
* If a search determines that no rows can be found regardless of
- index contents because the search conditions are contradictory (e.g.,
- x = 1 AND x = 2), then no predicate lock is needed.
+index contents because the search conditions are contradictory (e.g.,
+x = 1 AND x = 2), then no predicate lock is needed.
Other index AM implementation considerations:
* For an index AM that doesn't have support for predicate locking,
- we just acquire a predicate lock on the whole index for any search.
+we just acquire a predicate lock on the whole index for any search.
* B-tree index searches acquire predicate locks only on the
- index *leaf* pages needed to lock the appropriate index range. If,
- however, a search discovers that no root page has yet been created, a
- predicate lock on the index relation is required.
+index *leaf* pages needed to lock the appropriate index range. If,
+however, a search discovers that no root page has yet been created, a
+predicate lock on the index relation is required.
* Like a B-tree, GIN searches acquire predicate locks only on the
- leaf pages of entry tree. When performing an equality scan, and an
- entry has a posting tree, the posting tree root is locked instead, to
- lock only that key value. However, fastupdate=on postpones the
- insertion of tuples into index structure by temporarily storing them
- into pending list. That makes us unable to detect r-w conflicts using
- page-level locks. To cope with that, insertions to the pending list
- conflict with all scans.
+leaf pages of entry tree. When performing an equality scan, and an
+entry has a posting tree, the posting tree root is locked instead, to
+lock only that key value. However, fastupdate=on postpones the
+insertion of tuples into index structure by temporarily storing them
+into pending list. That makes us unable to detect r-w conflicts using
+page-level locks. To cope with that, insertions to the pending list
+conflict with all scans.
* GiST searches can determine that there are no matches at any
- level of the index, so we acquire predicate lock at each index
- level during a GiST search. An index insert at the leaf level can
- then be trusted to ripple up to all levels and locations where
- conflicting predicate locks may exist. In case there is a page split,
- we need to copy predicate lock from the original page to all the new
- pages.
+level of the index, so we acquire predicate lock at each index
+level during a GiST search. An index insert at the leaf level can
+then be trusted to ripple up to all levels and locations where
+conflicting predicate locks may exist. In case there is a page split,
+we need to copy predicate lock from the original page to all the new
+pages.
* Hash index searches acquire predicate locks on the primary
- page of a bucket. It acquires a lock on both the old and new buckets
- for scans that happen concurrently with page splits. During a bucket
- split, a predicate lock is copied from the primary page of an old
- bucket to the primary page of a new bucket.
+page of a bucket. It acquires a lock on both the old and new buckets
+for scans that happen concurrently with page splits. During a bucket
+split, a predicate lock is copied from the primary page of an old
+bucket to the primary page of a new bucket.
* The effects of page splits, overflows, consolidations, and
- removals must be carefully reviewed to ensure that predicate locks
- aren't "lost" during those operations, or kept with pages which could
- get re-used for different parts of the index.
+removals must be carefully reviewed to ensure that predicate locks
+aren't "lost" during those operations, or kept with pages which could
+get re-used for different parts of the index.
Innovations
@@ -448,34 +448,34 @@ Differences from the implementation described in the papers are
listed below.
* New structures needed to be created in shared memory to track
- the proper information for serializable transactions and their SIREAD
- locks.
+the proper information for serializable transactions and their SIREAD
+locks.
* Because PostgreSQL does not have the same concept of an "oldest
- transaction ID" for all serializable transactions as assumed in the
- Cahill thesis, we track the oldest snapshot xmin among serializable
- transactions, and a count of how many active transactions use that
- xmin. When the count hits zero we find the new oldest xmin and run a
- clean-up based on that.
+transaction ID" for all serializable transactions as assumed in the
+Cahill thesis, we track the oldest snapshot xmin among serializable
+transactions, and a count of how many active transactions use that
+xmin. When the count hits zero we find the new oldest xmin and run a
+clean-up based on that.
* Because reads in a subtransaction may cause that subtransaction
- to roll back, thereby affecting what is written by the top level
- transaction, predicate locks must survive a subtransaction rollback.
- As a consequence, all xid usage in SSI, including predicate locking,
- is based on the top level xid. When looking at an xid that comes
- from a tuple's xmin or xmax, for example, we always call
- SubTransGetTopmostTransaction() before doing much else with it.
+to roll back, thereby affecting what is written by the top level
+transaction, predicate locks must survive a subtransaction rollback.
+As a consequence, all xid usage in SSI, including predicate locking,
+is based on the top level xid. When looking at an xid that comes
+from a tuple's xmin or xmax, for example, we always call
+SubTransGetTopmostTransaction() before doing much else with it.
* PostgreSQL does not use "update in place" with a rollback log
- for its MVCC implementation. Where possible it uses "HOT" updates on
- the same page (if there is room and no indexed value is changed).
- For non-HOT updates the old tuple is expired in place and a new tuple
- is inserted at a new location. Because of this difference, a tuple
- lock in PostgreSQL doesn't automatically lock any other versions of a
- row. We don't try to copy or expand a tuple lock to any other
- versions of the row, based on the following proof that any additional
- serialization failures we would get from that would be false
- positives:
+for its MVCC implementation. Where possible it uses "HOT" updates on
+the same page (if there is room and no indexed value is changed).
+For non-HOT updates the old tuple is expired in place and a new tuple
+is inserted at a new location. Because of this difference, a tuple
+lock in PostgreSQL doesn't automatically lock any other versions of a
+row. We don't try to copy or expand a tuple lock to any other
+versions of the row, based on the following proof that any additional
+serialization failures we would get from that would be false
+positives:
- If transaction T1 reads a row version (thus acquiring a
predicate lock on it) and a second transaction T2 updates that row
@@ -595,33 +595,33 @@ This is intended to be the place to record specific issues which need
more detailed review or analysis.
* WAL file replay. While serializable implementations using S2PL
- can guarantee that the write-ahead log contains commits in a sequence
- consistent with some serial execution of serializable transactions,
- SSI cannot make that guarantee. While the WAL replay is no less
- consistent than under snapshot isolation, it is possible that under
- PITR recovery or hot standby a database could reach a readable state
- where some transactions appear before other transactions which would
- have had to precede them to maintain serializable consistency. In
- essence, if we do nothing, WAL replay will be at snapshot isolation
- even for serializable transactions. Is this OK? If not, how do we
- address it?
+can guarantee that the write-ahead log contains commits in a sequence
+consistent with some serial execution of serializable transactions,
+SSI cannot make that guarantee. While the WAL replay is no less
+consistent than under snapshot isolation, it is possible that under
+PITR recovery or hot standby a database could reach a readable state
+where some transactions appear before other transactions which would
+have had to precede them to maintain serializable consistency. In
+essence, if we do nothing, WAL replay will be at snapshot isolation
+even for serializable transactions. Is this OK? If not, how do we
+address it?
* External replication. Look at how this impacts external
- replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
- This is related to the "WAL file replay" issue.
+replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
+This is related to the "WAL file replay" issue.
* UNIQUE btree search for equality on all columns. Since a search
- of a UNIQUE index using equality tests on all columns will lock the
- heap tuple if an entry is found, it appears that there is no need to
- get a predicate lock on the index in that case. A predicate lock is
- still needed for such a search if a matching index entry which points
- to a visible tuple is not found.
+of a UNIQUE index using equality tests on all columns will lock the
+heap tuple if an entry is found, it appears that there is no need to
+get a predicate lock on the index in that case. A predicate lock is
+still needed for such a search if a matching index entry which points
+to a visible tuple is not found.
* Minimize touching of shared memory. Should lists in shared
- memory push entries which have just been returned to the front of the
- available list, so they will be popped back off soon and some memory
- might never be touched, or should we keep adding returned items to
- the end of the available list?
+memory push entries which have just been returned to the front of the
+available list, so they will be popped back off soon and some memory
+might never be touched, or should we keep adding returned items to
+the end of the available list?
References
diff --git a/src/backend/utils/mb/README b/src/backend/utils/mb/README
index afbe4c65d1..64de5b63ab 100644
--- a/src/backend/utils/mb/README
+++ b/src/backend/utils/mb/README
@@ -3,18 +3,18 @@ src/backend/utils/mb/README
Encodings
=========
- conv.c static functions and a public table for code conversion
- mbutils.c public functions for the backend only.
- stringinfo_mb.c public backend-only multibyte-aware stringinfo functions
- wstrcmp.c strcmp for mb
- wstrncmp.c strncmp for mb
+* conv.c static functions and a public table for code conversion
+* mbutils.c public functions for the backend only.
+* stringinfo_mb.c public backend-only multibyte-aware stringinfo functions
+* wstrcmp.c strcmp for mb
+* wstrncmp.c strncmp for mb
See also in src/common/:
- encnames.c public functions for encoding names
- wchar.c mostly static functions and a public table for mb string and
+* encnames.c public functions for encoding names
+* wchar.c mostly static functions and a public table for mb string and
multibyte conversion
Introduction
------------
- http://www.cprogramming.com/tutorial/unicode.html
+http://www.cprogramming.com/tutorial/unicode.html
diff --git a/src/tools/ci/README b/src/tools/ci/README
index e82073ff8a..f17ff3cbf7 100644
--- a/src/tools/ci/README
+++ b/src/tools/ci/README
@@ -35,7 +35,7 @@ See also https://cirrus-ci.org/guide/quick-start/
Once enabled on a repository, future commits and pull-requests in that
repository will automatically trigger CI builds. These are visible from the
commit history / PRs, and can also be viewed in the cirrus-ci UI at
-https://cirrus-ci.com/github/< username >/< reponame >/
+https://cirrus-ci.com/github/username/reponame/
Hint: all build log files are uploaded to cirrus-ci and can be downloaded
from the "Artifacts" section from the cirrus-ci UI after clicking into a
@@ -52,7 +52,7 @@ would need to build its own set of containers, which would be wasteful (both
in space and time.
These images are built, on a daily basis, from the specifications in
-github.com/anarazel/pg-vm-images/
+https://github.com/anarazel/pg-vm-images/
Controlling CI via commit messages
@@ -74,7 +74,7 @@ When running a lot of tests in a repository, cirrus-ci's free credits do not
suffice. In those cases a repository can be configured to use other
infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
variable can be configured for the repository in the cirrus-ci web interface,
-at https://cirrus-ci.com/github/< user or organization >. The file referenced
+at https://cirrus-ci.com/github/user_or_organization. The file referenced
(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
overwrite the default execution method for different operating systems,
defined in .cirrus.yml, by redefining the relevant yaml anchors.
--
2.39.3 (Apple Git-146)
[application/octet-stream] v4-0001-Standardize-syntax-in-internal-documentation.patch (144.6K, ../../[email protected]/3-v4-0001-Standardize-syntax-in-internal-documentation.patch)
download | inline diff:
From e68d4f6dd01cc2e65d2738d8a8f063c9c8470a54 Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <[email protected]>
Date: Tue, 10 Sep 2024 14:23:47 +0200
Subject: [PATCH v4 1/2] Standardize syntax in internal documentation
This patchset intends to achieve a greater degree of consistency
regarding the use of formatting whitespace and bullet list chars
and section indicators in the README files. As an added benefit
this will make most of the files render as Markdown, some syntax
violations remain as we aren't targeting a Markdown conversion.
Reviewed-by: Erik Wienhold <[email protected]>
Reviewed-by: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
contrib/start-scripts/macos/README | 7 +-
src/backend/access/gin/README | 170 +++----
src/backend/access/gist/README | 156 +++---
src/backend/access/hash/README | 76 +--
src/backend/access/heap/README.tuplock | 10 +-
src/backend/access/spgist/README | 80 +--
src/backend/access/transam/README | 56 +--
src/backend/lib/README | 22 +-
src/backend/libpq/README.SSL | 86 ++--
src/backend/optimizer/README | 244 ++++-----
src/backend/optimizer/plan/README | 112 ++---
src/backend/parser/README | 42 +-
src/backend/regex/README | 52 +-
src/backend/snowball/README | 14 +-
src/backend/storage/freespace/README | 56 +--
src/backend/storage/lmgr/README-SSI | 570 +++++++++++-----------
src/backend/utils/fmgr/README | 76 +--
src/backend/utils/mb/README | 16 +-
src/backend/utils/misc/README | 70 +--
src/backend/utils/mmgr/README | 10 +-
src/backend/utils/resowner/README | 52 +-
src/interfaces/ecpg/preproc/README.parser | 43 +-
src/port/README | 8 +-
src/test/isolation/README | 30 +-
src/test/kerberos/README | 4 +
src/test/locale/README | 3 +
src/test/modules/dummy_seclabel/README | 12 +-
src/test/modules/test_parser/README | 74 +--
src/test/modules/test_regex/README | 2 +-
src/test/modules/test_rls_hooks/README | 8 +-
src/test/modules/test_shm_mq/README | 14 +-
src/test/recovery/README | 4 +
src/test/ssl/README | 36 +-
src/timezone/tznames/README | 2 +-
src/tools/ci/README | 4 +-
src/tools/pgindent/README | 33 +-
36 files changed, 1172 insertions(+), 1082 deletions(-)
diff --git a/contrib/start-scripts/macos/README b/contrib/start-scripts/macos/README
index c4f2d9a270..c71f98880c 100644
--- a/contrib/start-scripts/macos/README
+++ b/contrib/start-scripts/macos/README
@@ -16,9 +16,12 @@ than "postgres", adjust the UserName parameter value for that.
4. Copy the modified org.postgresql.postgres.plist file into
/Library/LaunchDaemons/. You must do this as root:
- sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
-because the file will be ignored if it is not root-owned.
+
+ sudo cp org.postgresql.postgres.plist /Library/LaunchDaemons
+
+ because the file will be ignored if it is not root-owned.
At this point a reboot should launch the server. But if you want
to test it without rebooting, you can do
+
sudo launchctl load /Library/LaunchDaemons/org.postgresql.postgres.plist
diff --git a/src/backend/access/gin/README b/src/backend/access/gin/README
index b080731621..ccff689651 100644
--- a/src/backend/access/gin/README
+++ b/src/backend/access/gin/README
@@ -40,7 +40,7 @@ Core PostgreSQL includes built-in Gin support for one-dimensional arrays
Synopsis
--------
-=# create index txt_idx on aa using gin(a);
+ =# create index txt_idx on aa using gin(a);
Features
--------
@@ -111,40 +111,42 @@ by building a "normal" index tuple and then modifying it.) The points to
know are:
* In a single-column index, a key tuple just contains the key datum, but
-in a multi-column index, a key tuple contains the pair (column number,
-key datum) where the column number is stored as an int2. This is needed
-to support different key data types in different columns. This much of
-the tuple is built by index_form_tuple according to the usual rules.
-The column number (if present) can never be null, but the key datum can
-be, in which case a null bitmap is present as usual. (As usual for index
-tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
+ in a multi-column index, a key tuple contains the pair (column number,
+ key datum) where the column number is stored as an int2. This is needed
+ to support different key data types in different columns. This much of
+ the tuple is built by index_form_tuple according to the usual rules.
+ The column number (if present) can never be null, but the key datum can
+ be, in which case a null bitmap is present as usual. (As usual for index
+ tuples, the size of the null bitmap is fixed at INDEX_MAX_KEYS.)
* If the key datum is null (ie, IndexTupleHasNulls() is true), then
-just after the nominal index data (ie, at offset IndexInfoFindDataOffset
-or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
-the "category" of the null entry. These are the possible categories:
- 1 = ordinary null key value extracted from an indexable item
- 2 = placeholder for zero-key indexable item
- 3 = placeholder for null indexable item
-Placeholder null entries are inserted into the index because otherwise
-there would be no index entry at all for an empty or null indexable item,
-which would mean that full index scans couldn't be done and various corner
-cases would give wrong answers. The different categories of null entries
-are treated as distinct keys by the btree, but heap itempointers for the
-same category of null entry are merged into one index entry just as happens
-with ordinary key entries.
+ just after the nominal index data (ie, at offset IndexInfoFindDataOffset
+ or IndexInfoFindDataOffset + sizeof(int2)) there is a byte indicating
+ the "category" of the null entry. These are the possible categories:
+
+ 1 = ordinary null key value extracted from an indexable item
+ 2 = placeholder for zero-key indexable item
+ 3 = placeholder for null indexable item
+
+ Placeholder null entries are inserted into the index because otherwise
+ there would be no index entry at all for an empty or null indexable item,
+ which would mean that full index scans couldn't be done and various corner
+ cases would give wrong answers. The different categories of null entries
+ are treated as distinct keys by the btree, but heap itempointers for the
+ same category of null entry are merged into one index entry just as happens
+ with ordinary key entries.
* In a key entry at the btree leaf level, at the next SHORTALIGN boundary,
-there is a list of item pointers, in compressed format (see Posting List
-Compression section), pointing to the heap tuples for which the indexable
-items contain this key. This is called the "posting list".
-
-If the list would be too big for the index tuple to fit on an index page, the
-ItemPointers are pushed out to a separate posting page or pages, and none
-appear in the key entry itself. The separate pages are called a "posting
-tree" (see below); Note that in either case, the ItemPointers associated with
-a key can easily be read out in sorted order; this is relied on by the scan
-algorithms.
+ there is a list of item pointers, in compressed format (see Posting List
+ Compression section), pointing to the heap tuples for which the indexable
+ items contain this key. This is called the "posting list".
+
+ If the list would be too big for the index tuple to fit on an index page, the
+ ItemPointers are pushed out to a separate posting page or pages, and none
+ appear in the key entry itself. The separate pages are called a "posting
+ tree" (see below); Note that in either case, the ItemPointers associated with
+ a key can easily be read out in sorted order; this is relied on by the scan
+ algorithms.
* The index tuple header fields of a leaf key entry are abused as follows:
@@ -325,11 +327,11 @@ getting them on the next page.
The picture below shows tree state after finding the leaf page. Lower case
letters depicts tree pages. 'S' depicts shared lock on the page.
- a
- / | \
- b c d
- / | \ | \ | \
- eS f g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS f g h i j k
### Steping right
@@ -346,11 +348,11 @@ concurrently and doesn't delete right sibling accordingly.
The picture below shows two pages locked at once during stepping right.
- a
- / | \
- b c d
- / | \ | \ | \
- eS fS g h i j k
+ a
+ / | \
+ b c d
+ / | \ | \ | \
+ eS fS g h i j k
### Insert
@@ -365,11 +367,11 @@ The picture below shows leaf page locked in exclusive mode and ready for
insertion. 'P' and 'E' depict pin and exclusive lock correspondingly.
- aP
- / | \
- b cP d
- / | \ | \ | \
- e f g hE i j k
+ aP
+ / | \
+ b cP d
+ / | \ | \ | \
+ e f g hE i j k
If insert causes a page split, the parent is locked in exclusive mode before
@@ -379,11 +381,11 @@ parent and child pages at once starting from child.
The picture below shows tree state after leaf page split. 'q' is new page
produced by split. Parent 'c' is about to have downlink inserted.
- aP
- / | \
- b cE d
- / | \ / | \ | \
- e f g hE q i j k
+ aP
+ / | \
+ b cE d
+ / | \ / | \ | \
+ e f g hE q i j k
### Page deletion
@@ -404,11 +406,11 @@ we locked it.
The picture below shows tree state after page deletion algorithm traversed to
leftmost leaf of the tree.
- aE
- / | \
- bE c d
- / | \ | \ | \
- eE f g h i j k
+ aE
+ / | \
+ bE c d
+ / | \ | \ | \
+ eE f g h i j k
Deletion algorithm keeps exclusive locks on left siblings of pages comprising
currently investigated path. Thus, if current page is to be removed, all
@@ -436,21 +438,21 @@ The picture below shows tree state after page deletion algorithm further
traversed the tree. Currently investigated path is 'a-c-h'. Left siblings 'b'
and 'g' of 'c' and 'h' correspondingly are also exclusively locked.
- aE
- / | \
- bE cE d
- / | \ | \ | \
- e f gE hE i j k
+ aE
+ / | \
+ bE cE d
+ / | \ | \ | \
+ e f gE hE i j k
The next picture shows tree state after page 'h' was deleted. It's marked with
'deleted' flag and newest xid, which might visit it. Downlink from 'c' to 'h'
is also deleted.
- aE
- / | \
- bE cE d
- / | \ \ | \
- e f gE hD iE j k
+ aE
+ / | \
+ bE cE d
+ / | \ \ | \
+ e f gE hD iE j k
However, it's still possible that concurrent reader has seen downlink from 'c'
to 'h' before we deleted it. In that case this reader will step right from 'h'
@@ -463,11 +465,11 @@ The next picture shows tree state after 'i' and 'c' was deleted. Internal page
investigation is 'a-d-j'. Pages 'b' and 'g' are locked as self siblings of 'd'
and 'j'.
- aE
- / \
- bE cD dE
- / | \ | \
- e f gE hD iD jE k
+ aE
+ / \
+ bE cD dE
+ / | \ | \
+ e f gE hD iD jE k
During the replay of page deletion at standby, the page's left sibling, the
target page, and its parent, are locked in that order. This order guarantees
@@ -516,44 +518,44 @@ For compatibility, old uncompressed format is also supported. Following
rules are used to handle it:
* GIN_ITUP_COMPRESSED flag marks index tuples that contain a posting list.
-This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
-Use GinItupIsCompressed(itup) to check the flag.
+ This flag is stored in high bit of ItemPointerGetBlockNumber(&itup->t_tid).
+ Use GinItupIsCompressed(itup) to check the flag.
* Posting tree pages in the new format are marked with the GIN_COMPRESSED flag.
Macros GinPageIsCompressed(page) and GinPageSetCompressed(page) are used to
check and set this flag.
* All scan operations check format of posting list add use corresponding code
-to read its content.
+ to read its content.
* When updating an index tuple containing an uncompressed posting list, it
-will be replaced with new index tuple containing a compressed list.
+ will be replaced with new index tuple containing a compressed list.
* When updating an uncompressed posting tree leaf page, it's compressed.
* If vacuum finds some dead TIDs in uncompressed posting lists, they are
-converted into compressed posting lists. This assumes that the compressed
-posting list fits in the space occupied by the uncompressed list. IOW, we
-assume that the compressed version of the page, with the dead items removed,
-takes less space than the old uncompressed version.
+ converted into compressed posting lists. This assumes that the compressed
+ posting list fits in the space occupied by the uncompressed list. IOW, we
+ assume that the compressed version of the page, with the dead items removed,
+ takes less space than the old uncompressed version.
Limitations
-----------
- * Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
- * Gin searches entries only by equality matching, or simple range
- matching using the "partial match" feature.
+* Gin doesn't use scan->kill_prior_tuple & scan->ignore_killed_tuples
+* Gin searches entries only by equality matching, or simple range
+ matching using the "partial match" feature.
TODO
----
Nearest future:
- * Opclasses for more types (no programming, just many catalog changes)
+* Opclasses for more types (no programming, just many catalog changes)
Distant future:
- * Replace B-tree of entries to something like GiST
+* Replace B-tree of entries to something like GiST
Authors
-------
diff --git a/src/backend/access/gist/README b/src/backend/access/gist/README
index 8015ff19f0..62818b09d7 100644
--- a/src/backend/access/gist/README
+++ b/src/backend/access/gist/README
@@ -183,70 +183,70 @@ operation.
findPath is a subroutine of findParent, used when the correct parent page
can't be found by following the rightlinks at the parent level:
-findPath( stack item )
- push stack, [root, 0, 0] // page, LSN, parent
- while( stack )
- ptr = top of stack
- latch( ptr->page, S-mode )
- if ( ptr->parent->page->lsn < ptr->page->nsn )
- push stack, [ ptr->page->rightlink, 0, ptr->parent ]
- end
- for( each tuple on page )
- if ( tuple->pagepointer == item->page )
- return stack
- else
- add to stack at the end [tuple->pagepointer,0, ptr]
+ findPath( stack item )
+ push stack, [root, 0, 0] // page, LSN, parent
+ while( stack )
+ ptr = top of stack
+ latch( ptr->page, S-mode )
+ if ( ptr->parent->page->lsn < ptr->page->nsn )
+ push stack, [ ptr->page->rightlink, 0, ptr->parent ]
+ end
+ for( each tuple on page )
+ if ( tuple->pagepointer == item->page )
+ return stack
+ else
+ add to stack at the end [tuple->pagepointer,0, ptr]
+ end
end
+ unlatch( ptr->page )
+ pop stack
end
- unlatch( ptr->page )
- pop stack
- end
gistFindCorrectParent is used to re-find the parent of a page during
insertion. It might have migrated to the right since we traversed down the
tree because of page splits.
-findParent( stack item )
- parent = item->parent
- if ( parent->page->lsn != parent->lsn )
- while(true)
- search parent tuple on parent->page, if found the return
- rightlink = parent->page->rightlink
- unlatch( parent->page )
- if ( rightlink is incorrect )
- break loop
+ findParent( stack item )
+ parent = item->parent
+ if ( parent->page->lsn != parent->lsn )
+ while(true)
+ search parent tuple on parent->page, if found the return
+ rightlink = parent->page->rightlink
+ unlatch( parent->page )
+ if ( rightlink is incorrect )
+ break loop
+ end
+ parent->page = rightlink
+ latch( parent->page, X-mode )
end
- parent->page = rightlink
+ newstack = findPath( item->parent )
+ replace part of stack to new one
latch( parent->page, X-mode )
+ return findParent( item )
end
- newstack = findPath( item->parent )
- replace part of stack to new one
- latch( parent->page, X-mode )
- return findParent( item )
- end
pageSplit function decides how to distribute keys to the new pages after
page split:
-pageSplit(page, allkeys)
- (lkeys, rkeys) = pickSplit( allkeys )
- if ( page is root )
- lpage = new page
- else
- lpage = page
- rpage = new page
- if ( no space left on rpage )
- newkeys = pageSplit( rpage, rkeys )
- else
- push newkeys, union(rkeys)
- end
- if ( no space left on lpage )
- push newkeys, pageSplit( lpage, lkeys )
- else
- push newkeys, union(lkeys)
- end
- return newkeys
+ pageSplit(page, allkeys)
+ (lkeys, rkeys) = pickSplit( allkeys )
+ if ( page is root )
+ lpage = new page
+ else
+ lpage = page
+ rpage = new page
+ if ( no space left on rpage )
+ newkeys = pageSplit( rpage, rkeys )
+ else
+ push newkeys, union(rkeys)
+ end
+ if ( no space left on lpage )
+ push newkeys, pageSplit( lpage, lkeys )
+ else
+ push newkeys, union(lkeys)
+ end
+ return newkeys
@@ -302,18 +302,18 @@ In the algorithm, levels are numbered so that leaf pages have level zero,
and internal node levels count up from 1. This numbering ensures that a page's
level number never changes, even when the root page is split.
-Level Tree
+ Level Tree
-3 *
- / \
-2 * *
- / | \ / | \
-1 * * * * * *
- / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o
+ 3 *
+ / \
+ 2 * *
+ / | \ / | \
+ 1 * * * * * *
+ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o
-* - internal page
-o - leaf page
+ * - internal page
+ o - leaf page
Internal pages that belong to certain levels have buffers associated with
them. Leaf pages never have buffers. Which levels have buffers is controlled
@@ -322,17 +322,17 @@ have buffers, while others do not. For example, if level_step = 2, then
pages on levels 2, 4, 6, ... have buffers. If level_step = 1 then every
internal page has a buffer.
-Level Tree (level_step = 1) Tree (level_step = 2)
+ Level Tree (level_step = 1) Tree (level_step = 2)
-3 * *
- / \ / \
-2 *(b) *(b) *(b) *(b)
- / | \ / | \ / | \ / | \
-1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
- / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
-0 o o o o o o o o o o o o o o o o o o o o o o o o
+ 3 * *
+ / \ / \
+ 2 *(b) *(b) *(b) *(b)
+ / | \ / | \ / | \ / | \
+ 1 *(b) *(b) *(b) *(b) *(b) *(b) * * * * * *
+ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \
+ 0 o o o o o o o o o o o o o o o o o o o o o o o o
-(b) - buffer
+ (b) - buffer
Logically, a buffer is just bunch of tuples. Physically, it is divided in
pages, backed by a temporary file. Each buffer can be in one of two states:
@@ -365,7 +365,7 @@ pages where the tuples finally land on get cached too. If there are, the last
buffer page of each buffer below is kept in memory. This is illustrated in
the figures below:
- Buffer being emptied to
+ Buffer being emptied to
lower-level buffers Buffer being emptied to leaf pages
+(fb) +(fb)
@@ -374,11 +374,11 @@ the figures below:
/ \ / \ / \ / \
*(ab) *(ab) *(ab) *(ab) x x x x
-+ - cached internal page
-x - cached leaf page
-* - non-cached internal page
-(fb) - buffer being emptied
-(ab) - buffers being appended to, with last page in memory
+ + - cached internal page
+ x - cached leaf page
+ * - non-cached internal page
+ (fb) - buffer being emptied
+ (ab) - buffers being appended to, with last page in memory
In the beginning of the index build, the level-step is chosen so that all those
pages involved in emptying one buffer fit in cache, so after each of those
@@ -477,7 +477,7 @@ value. The page is not recycled, until that XID is no longer visible to
anyone. That's much more conservative than necessary, but let's keep it
simple.
-
-Authors:
- Teodor Sigaev <[email protected]>
- Oleg Bartunov <[email protected]>
+Authors
+-------
+* Teodor Sigaev <[email protected]>
+* Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/hash/README b/src/backend/access/hash/README
index 13dc59c124..5b4c758df6 100644
--- a/src/backend/access/hash/README
+++ b/src/backend/access/hash/README
@@ -248,24 +248,24 @@ track of available overflow pages.
The reader algorithm is:
- lock the primary bucket page of the target bucket
- if the target bucket is still being populated by a split:
- release the buffer content lock on current bucket page
- pin and acquire the buffer content lock on old bucket in shared mode
- release the buffer content lock on old bucket, but not pin
- retake the buffer content lock on new bucket
- arrange to scan the old bucket normally and the new bucket for
- tuples which are not moved-by-split
--- then, per read request:
- reacquire content lock on current page
- step to next page if necessary (no chaining of content locks, but keep
- the pin on the primary bucket throughout the scan)
- save all the matching tuples from current index page into an items array
- release pin and content lock (but if it is primary bucket page retain
- its pin till the end of the scan)
- get tuple from an item array
--- at scan shutdown:
- release all pins still held
+ lock the primary bucket page of the target bucket
+ if the target bucket is still being populated by a split:
+ release the buffer content lock on current bucket page
+ pin and acquire the buffer content lock on old bucket in shared mode
+ release the buffer content lock on old bucket, but not pin
+ retake the buffer content lock on new bucket
+ arrange to scan the old bucket normally and the new bucket for
+ tuples which are not moved-by-split
+ -- then, per read request:
+ reacquire content lock on current page
+ step to next page if necessary (no chaining of content locks, but keep
+ the pin on the primary bucket throughout the scan)
+ save all the matching tuples from current index page into an items array
+ release pin and content lock (but if it is primary bucket page retain
+ its pin till the end of the scan)
+ get tuple from an item array
+ -- at scan shutdown:
+ release all pins still held
Holding the buffer pin on the primary bucket page for the whole scan prevents
the reader's current-tuple pointer from being invalidated by splits or
@@ -288,8 +288,8 @@ which this bucket is formed by split.
The insertion algorithm is rather similar:
lock the primary bucket page of the target bucket
--- (so far same as reader, except for acquisition of buffer content lock in
- exclusive mode on primary bucket page)
+ -- (so far same as reader, except for acquisition of buffer content lock in
+ exclusive mode on primary bucket page)
if the bucket-being-split flag is set for a bucket and pin count on it is
one, then finish the split
release the buffer content lock on current bucket
@@ -465,24 +465,24 @@ overflow page to the free pool.
Obtaining an overflow page:
- take metapage content lock in exclusive mode
- determine next bitmap page number; if none, exit loop
- release meta page content lock
- pin bitmap page and take content lock in exclusive mode
- search for a free page (zero bit in bitmap)
- if found:
- set bit in bitmap
- mark bitmap page dirty
- take metapage buffer content lock in exclusive mode
- if first-free-bit value did not change,
- update it and mark meta page dirty
- else (not found):
- release bitmap page buffer content lock
- loop back to try next bitmap page, if any
--- here when we have checked all bitmap pages; we hold meta excl. lock
- extend index to add another overflow page; update meta information
- mark meta page dirty
- return page number
+ take metapage content lock in exclusive mode
+ determine next bitmap page number; if none, exit loop
+ release meta page content lock
+ pin bitmap page and take content lock in exclusive mode
+ search for a free page (zero bit in bitmap)
+ if found:
+ set bit in bitmap
+ mark bitmap page dirty
+ take metapage buffer content lock in exclusive mode
+ if first-free-bit value did not change,
+ update it and mark meta page dirty
+ else (not found):
+ release bitmap page buffer content lock
+ loop back to try next bitmap page, if any
+ -- here when we have checked all bitmap pages; we hold meta excl. lock
+ extend index to add another overflow page; update meta information
+ mark meta page dirty
+ return page number
It is slightly annoying to release and reacquire the metapage lock
multiple times, but it seems best to do it that way to minimize loss of
diff --git a/src/backend/access/heap/README.tuplock b/src/backend/access/heap/README.tuplock
index 818cd7f980..df785fdb83 100644
--- a/src/backend/access/heap/README.tuplock
+++ b/src/backend/access/heap/README.tuplock
@@ -62,11 +62,11 @@ the tuple without changing its key.
The conflict table is:
- UPDATE NO KEY UPDATE SHARE KEY SHARE
-UPDATE conflict conflict conflict conflict
-NO KEY UPDATE conflict conflict conflict
-SHARE conflict conflict
-KEY SHARE conflict
+ UPDATE NO KEY UPDATE SHARE KEY SHARE
+ UPDATE conflict conflict conflict conflict
+ NO KEY UPDATE conflict conflict conflict
+ SHARE conflict conflict
+ KEY SHARE conflict
When there is a single locker in a tuple, we can just store the locking info
in the tuple itself. We do this by storing the locker's Xid in XMAX, and
diff --git a/src/backend/access/spgist/README b/src/backend/access/spgist/README
index 7117e02c77..2b890e4f8e 100644
--- a/src/backend/access/spgist/README
+++ b/src/backend/access/spgist/README
@@ -13,7 +13,7 @@ few disk pages, even if it traverses many nodes.
COMMON STRUCTURE DESCRIPTION
-
+----------------------------
Logically, an SP-GiST tree is a set of tuples, each of which can be either
an inner or leaf tuple. Each inner tuple contains "nodes", which are
(label,pointer) pairs, where the pointer (ItemPointerData) is a pointer to
@@ -58,27 +58,27 @@ pages.
An inner tuple consists of:
- optional prefix value - all successors must be consistent with it.
- Example:
- radix tree - prefix value is a common prefix string
- quad tree - centroid
- k-d tree - one coordinate
+ optional prefix value - all successors must be consistent with it.
+ Example:
+ radix tree - prefix value is a common prefix string
+ quad tree - centroid
+ k-d tree - one coordinate
- list of nodes, where node is a (label, pointer) pair.
- Example of a label: a single character for radix tree
+ list of nodes, where node is a (label, pointer) pair.
+ Example of a label: a single character for radix tree
A leaf tuple consists of:
- a leaf value
- Example:
- radix tree - the rest of string (postfix)
- quad and k-d tree - the point itself
+ a leaf value
+ Example:
+ radix tree - the rest of string (postfix)
+ quad and k-d tree - the point itself
- ItemPointer to the corresponding heap tuple
- nextOffset number of next leaf tuple in a chain on a leaf page
+ ItemPointer to the corresponding heap tuple
+ nextOffset number of next leaf tuple in a chain on a leaf page
- optional nulls bitmask
- optional INCLUDE-column values
+ optional nulls bitmask
+ optional INCLUDE-column values
For compatibility with pre-v14 indexes, a leaf tuple has a nulls bitmask
only if there are null values (among the leaf value and the INCLUDE values)
@@ -90,7 +90,7 @@ code can be used.
NULLS HANDLING
-
+--------------
We assume that SPGiST-indexable operators are strict (can never succeed for
null inputs). It is still desirable to index nulls, so that whole-table
indexscans are possible and so that "x IS NULL" can be implemented by an
@@ -104,27 +104,27 @@ AllTheSame cases in the normal tree.
INSERTION ALGORITHM
-
+-------------------
Insertion algorithm is designed to keep the tree in a consistent state at
any moment. Here is a simplified insertion algorithm specification
(numbers refer to notes below):
- Start with the first tuple on the root page (1)
-
- loop:
- if (page is leaf) then
- if (enough space)
- insert on page and exit (5)
- else (7)
- call PickSplitFn() (2)
- end if
- else
- switch (chooseFn())
- case MatchNode - descend through selected node
- case AddNode - add node and then retry chooseFn (3, 6)
- case SplitTuple - split inner tuple to prefix and postfix, then
- retry chooseFn with the prefix tuple (4, 6)
- end if
+ Start with the first tuple on the root page (1)
+
+ loop:
+ if (page is leaf) then
+ if (enough space)
+ insert on page and exit (5)
+ else (7)
+ call PickSplitFn() (2)
+ end if
+ else
+ switch (chooseFn())
+ case MatchNode - descend through selected node
+ case AddNode - add node and then retry chooseFn (3, 6)
+ case SplitTuple - split inner tuple to prefix and postfix, then
+ retry chooseFn with the prefix tuple (4, 6)
+ end if
Notes:
@@ -160,7 +160,7 @@ the following notation, where tuple's id is just for discussion (no such id
is actually stored):
inner tuple: {tuple id}(prefix string)[ comma separated list of node labels ]
-leaf tuple: {tuple id}<value>
+leaf tuple: {tuple id}< value >
Suppose we need to insert string 'www.gogo.com' into inner tuple
@@ -215,7 +215,7 @@ space utilization, but doesn't change the basis of the algorithm.
CONCURRENCY
-
+-----------
While descending the tree, the insertion algorithm holds exclusive lock on
two tree levels at a time, ie both parent and child pages (but parent and
child pages can be the same, see notes above). There is a possibility of
@@ -267,7 +267,7 @@ been flushed out of the system.
DEAD TUPLES
-
+-----------
Tuples on leaf pages can be in one of four states:
SPGIST_LIVE: normal, live pointer to a heap tuple.
@@ -319,7 +319,7 @@ remove unused inner tuples.
VACUUM
-
+------
VACUUM (or more precisely, spgbulkdelete) performs a single sequential scan
over the entire index. On both leaf and inner pages, we can convert old
REDIRECT tuples into PLACEHOLDER status, and then remove any PLACEHOLDERs
@@ -374,7 +374,7 @@ space map, and gather statistics.
LAST USED PAGE MANAGEMENT
-
+-------------------------
The list of last used pages contains four pages - a leaf page and three
inner pages, one from each "triple parity" group. (Actually, there's one
such list for the main tree and a separate one for the nulls tree.) This
@@ -384,6 +384,6 @@ critical, because we could allocate a new page at any moment.
AUTHORS
-
+-------
Teodor Sigaev <[email protected]>
Oleg Bartunov <[email protected]>
diff --git a/src/backend/access/transam/README b/src/backend/access/transam/README
index 6e4711dace..62015da1fe 100644
--- a/src/backend/access/transam/README
+++ b/src/backend/access/transam/README
@@ -59,27 +59,27 @@ For example, consider the following sequence of user commands:
In the main processing loop, this results in the following function call
sequence:
- / StartTransactionCommand;
- / StartTransaction;
-1) < ProcessUtility; << BEGIN
- \ BeginTransactionBlock;
- \ CommitTransactionCommand;
-
- / StartTransactionCommand;
-2) / PortalRunSelect; << SELECT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
-3) / ProcessQuery; << INSERT ...
- \ CommitTransactionCommand;
- \ CommandCounterIncrement;
-
- / StartTransactionCommand;
- / ProcessUtility; << COMMIT
-4) < EndTransactionBlock;
- \ CommitTransactionCommand;
- \ CommitTransaction;
+ / StartTransactionCommand;
+ / StartTransaction;
+ 1) < ProcessUtility; << BEGIN
+ \ BeginTransactionBlock;
+ \ CommitTransactionCommand;
+
+ / StartTransactionCommand;
+ 2) / PortalRunSelect; << SELECT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ 3) / ProcessQuery; << INSERT ...
+ \ CommitTransactionCommand;
+ \ CommandCounterIncrement;
+
+ / StartTransactionCommand;
+ / ProcessUtility; << COMMIT
+ 4) < EndTransactionBlock;
+ \ CommitTransactionCommand;
+ \ CommitTransaction;
The point of this example is to demonstrate the need for
StartTransactionCommand and CommitTransactionCommand to be state smart -- they
@@ -100,12 +100,12 @@ Transaction aborts can occur in two ways:
The reason we have to distinguish them is illustrated by the following two
situations:
- case 1 case 2
- ------ ------
-1) user types BEGIN 1) user types BEGIN
-2) user does something 2) user does something
-3) user does not like what 3) system aborts for some reason
- she sees and types ABORT (syntax error, etc)
+ case 1 case 2
+ ------ ------
+ 1) user types BEGIN 1) user types BEGIN
+ 2) user does something 2) user does something
+ 3) user does not like what 3) system aborts for some reason
+ she sees and types ABORT (syntax error, etc)
In case 1, we want to abort the transaction and return to the default state.
In case 2, there may be more commands coming our way which are part of the
@@ -171,7 +171,7 @@ CommitTransactionCommand, the real work is done. The main point of doing
things this way is that if we get an error while popping state stack entries,
the remaining stack entries still show what we need to do to finish up.
-In the case of ROLLBACK TO <savepoint>, we abort all the subtransactions up
+In the case of ROLLBACK TO < savepoint >, we abort all the subtransactions up
through the one identified by the savepoint name, and then re-create that
subtransaction level with the same name. So it's a completely new
subtransaction as far as the internals are concerned.
diff --git a/src/backend/lib/README b/src/backend/lib/README
index f2fb591237..735e379051 100644
--- a/src/backend/lib/README
+++ b/src/backend/lib/README
@@ -1,27 +1,27 @@
This directory contains a general purpose data structures, for use anywhere
in the backend:
-binaryheap.c - a binary heap
+* binaryheap.c - a binary heap
-bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
+* bipartite_match.c - Hopcroft-Karp maximum cardinality algorithm for bipartite graphs
-bloomfilter.c - probabilistic, space-efficient set membership testing
+* bloomfilter.c - probabilistic, space-efficient set membership testing
-dshash.c - concurrent hash tables backed by dynamic shared memory areas
+* dshash.c - concurrent hash tables backed by dynamic shared memory areas
-hyperloglog.c - a streaming cardinality estimator
+* hyperloglog.c - a streaming cardinality estimator
-ilist.c - single and double-linked lists
+* ilist.c - single and double-linked lists
-integerset.c - a data structure for holding large set of integers
+* integerset.c - a data structure for holding large set of integers
-knapsack.c - knapsack problem solver
+* knapsack.c - knapsack problem solver
-pairingheap.c - a pairing heap
+* pairingheap.c - a pairing heap
-rbtree.c - a red-black tree
+* rbtree.c - a red-black tree
-stringinfo.c - an extensible string type
+* stringinfo.c - an extensible string type
Aside from the inherent characteristics of the data structures, there are a
diff --git a/src/backend/libpq/README.SSL b/src/backend/libpq/README.SSL
index d84a434a6e..98e20498bd 100644
--- a/src/backend/libpq/README.SSL
+++ b/src/backend/libpq/README.SSL
@@ -3,59 +3,59 @@ src/backend/libpq/README.SSL
SSL
===
->From the servers perspective:
+From the servers perspective:
- Receives StartupPacket
- |
- |
- (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
- | No
- |
- | Yes
+ Receives StartupPacket
+ |
+ |
+ (Is SSL_NEGOTIATE_CODE?) ----------- Normal startup
+ | No
+ |
+ | Yes
+ |
+ |
+ (Server compiled with USE_SSL?) ------- Send 'N'
+ | No |
+ | |
+ | Yes Normal startup
+ |
+ |
+ Send 'S'
+ |
+ |
+ Establish SSL
+ |
+ |
+ Normal startup
+
+
+
+
+
+From the clients perspective (v6.6 client _with_ SSL):
+
+
+ Connect
|
|
- (Server compiled with USE_SSL?) ------- Send 'N'
- | No |
- | |
- | Yes Normal startup
+ Send packet with SSL_NEGOTIATE_CODE
|
|
- Send 'S'
+ Receive single char ------- 'S' -------- Establish SSL
+ | |
+ | '<else>' |
+ | Normal startup
|
|
- Establish SSL
+ Is it 'E' for error ------------------- Retry connection
+ | Yes without SSL
+ | No
|
+ Is it 'N' for normal ------------------- Normal startup
+ | Yes
|
- Normal startup
-
-
-
-
-
->From the clients perspective (v6.6 client _with_ SSL):
-
-
- Connect
- |
- |
- Send packet with SSL_NEGOTIATE_CODE
- |
- |
- Receive single char ------- 'S' -------- Establish SSL
- | |
- | '<else>' |
- | Normal startup
- |
- |
- Is it 'E' for error ------------------- Retry connection
- | Yes without SSL
- | No
- |
- Is it 'N' for normal ------------------- Normal startup
- | Yes
- |
- Fail with unknown
+ Fail with unknown
---------------------------------------------------------------------------
diff --git a/src/backend/optimizer/README b/src/backend/optimizer/README
index 2ab4f3dbf3..5805b61a23 100644
--- a/src/backend/optimizer/README
+++ b/src/backend/optimizer/README
@@ -254,7 +254,9 @@ the boundary, unless the proposed join is a LEFT join that can associate
into the SpecialJoinInfo's RHS using identity 3.
The use of minimum Relid sets has some pitfalls; consider a query like
+
A leftjoin (B leftjoin (C innerjoin D) on (Pbcd)) on Pa
+
where Pa doesn't mention B/C/D at all. In this case a naive computation
would give the upper leftjoin's min LHS as {A} and min RHS as {C,D} (since
we know that the innerjoin can't associate out of the leftjoin's RHS, and
@@ -262,7 +264,9 @@ enforce that by including its relids in the leftjoin's min RHS). And the
lower leftjoin has min LHS of {B} and min RHS of {C,D}. Given such
information, join_is_legal would think it's okay to associate the upper
join into the lower join's RHS, transforming the query to
+
B leftjoin (A leftjoin (C innerjoin D) on Pa) on (Pbcd)
+
which yields totally wrong answers. We prevent that by forcing the min RHS
for the upper join to include B. This is perhaps overly restrictive, but
such cases don't arise often so it's not clear that it's worth developing a
@@ -359,10 +363,12 @@ side of the full join a Var came from; but that information can be found
elsewhere at need.)
Notionally, a Var having nonempty varnullingrels can be thought of as
+
CASE WHEN any-of-these-outer-joins-produced-a-null-extended-row
THEN NULL
ELSE the-scan-level-value-of-the-column
END
+
It's only notional, because no such calculation is ever done explicitly.
In a finished plan, Vars occurring in scan-level plan nodes represent
the actual table column values, but upper-level Vars are always
@@ -375,14 +381,20 @@ otherwise be essential information for FULL JOIN cases.
Outer join identity 3 (discussed above) complicates this picture
a bit. In the form
+
A leftjoin (B leftjoin C on (Pbc)) on (Pab)
+
all of the Vars in clauses Pbc and Pab will have empty varnullingrels,
but if we start with
+
(A leftjoin B on (Pab)) leftjoin C on (Pbc)
+
then the parser will have marked Pbc's B Vars with the A/B join's
RT index, making this form artificially different from the first.
For discussion's sake, let's denote this marking with a star:
+
(A leftjoin B on (Pab)) leftjoin C on (Pb*c)
+
To cope with this, once we have detected that commuting these joins
is legal, we generate both the Pbc and Pb*c forms of that ON clause,
by either removing or adding the first join's RT index in the B Vars
@@ -553,114 +565,114 @@ Optimizer Functions
The primary entry point is planner().
-planner()
-set up for recursive handling of subqueries
--subquery_planner()
- pull up sublinks and subqueries from rangetable, if possible
- canonicalize qual
- Attempt to simplify WHERE clause to the most useful form; this includes
- flattening nested AND/ORs and detecting clauses that are duplicated in
- different branches of an OR.
- simplify constant expressions
- process sublinks
- convert Vars of outer query levels into Params
---grouping_planner()
- preprocess target list for non-SELECT queries
- handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
- ORDER BY, DISTINCT, LIMIT
----query_planner()
- make list of base relations used in query
- split up the qual into restrictions (a=1) and joins (b=c)
- find qual clauses that enable merge and hash joins
-----make_one_rel()
- set_base_rel_pathlists()
- find seqscan and all index paths for each base relation
- find selectivity of columns used in joins
- make_rel_from_joinlist()
- hand off join subproblems to a plugin, GEQO, or standard_join_search()
-------standard_join_search()
- call join_search_one_level() for each level of join tree needed
- join_search_one_level():
- For each joinrel of the prior level, do make_rels_by_clause_joins()
- if it has join clauses, or make_rels_by_clauseless_joins() if not.
- Also generate "bushy plan" joins between joinrels of lower levels.
- Back at standard_join_search(), generate gather paths if needed for
- each newly constructed joinrel, then apply set_cheapest() to extract
- the cheapest path for it.
- Loop back if this wasn't the top join level.
- Back at grouping_planner:
- do grouping (GROUP BY) and aggregation
- do window functions
- make unique (DISTINCT)
- do sorting (ORDER BY)
- do limit (LIMIT/OFFSET)
-Back at planner():
-convert finished Path tree into a Plan tree
-do final cleanup after planning
+ planner()
+ set up for recursive handling of subqueries
+ -subquery_planner()
+ pull up sublinks and subqueries from rangetable, if possible
+ canonicalize qual
+ Attempt to simplify WHERE clause to the most useful form; this includes
+ flattening nested AND/ORs and detecting clauses that are duplicated in
+ different branches of an OR.
+ simplify constant expressions
+ process sublinks
+ convert Vars of outer query levels into Params
+ --grouping_planner()
+ preprocess target list for non-SELECT queries
+ handle UNION/INTERSECT/EXCEPT, GROUP BY, HAVING, aggregates,
+ ORDER BY, DISTINCT, LIMIT
+ ---query_planner()
+ make list of base relations used in query
+ split up the qual into restrictions (a=1) and joins (b=c)
+ find qual clauses that enable merge and hash joins
+ ----make_one_rel()
+ set_base_rel_pathlists()
+ find seqscan and all index paths for each base relation
+ find selectivity of columns used in joins
+ make_rel_from_joinlist()
+ hand off join subproblems to a plugin, GEQO, or standard_join_search()
+ ------standard_join_search()
+ call join_search_one_level() for each level of join tree needed
+ join_search_one_level():
+ For each joinrel of the prior level, do make_rels_by_clause_joins()
+ if it has join clauses, or make_rels_by_clauseless_joins() if not.
+ Also generate "bushy plan" joins between joinrels of lower levels.
+ Back at standard_join_search(), generate gather paths if needed for
+ each newly constructed joinrel, then apply set_cheapest() to extract
+ the cheapest path for it.
+ Loop back if this wasn't the top join level.
+ Back at grouping_planner:
+ do grouping (GROUP BY) and aggregation
+ do window functions
+ make unique (DISTINCT)
+ do sorting (ORDER BY)
+ do limit (LIMIT/OFFSET)
+ Back at planner():
+ convert finished Path tree into a Plan tree
+ do final cleanup after planning
Optimizer Data Structures
-------------------------
-PlannerGlobal - global information for a single planner invocation
-
-PlannerInfo - information for planning a particular Query (we make
- a separate PlannerInfo node for each sub-Query)
-
-RelOptInfo - a relation or joined relations
-
- RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
- (note the same structure is used for restriction and
- join clauses)
-
- Path - every way to generate a RelOptInfo(sequential,index,joins)
- A plain Path node can represent several simple plans, per its pathtype:
- T_SeqScan - sequential scan
- T_SampleScan - tablesample scan
- T_FunctionScan - function-in-FROM scan
- T_TableFuncScan - table function scan
- T_ValuesScan - VALUES scan
- T_CteScan - CTE (WITH) scan
- T_NamedTuplestoreScan - ENR scan
- T_WorkTableScan - scan worktable of a recursive CTE
- T_Result - childless Result plan node (used for FROM-less SELECT)
- IndexPath - index scan
- BitmapHeapPath - top of a bitmapped index scan
- TidPath - scan by CTID
- TidRangePath - scan a contiguous range of CTIDs
- SubqueryScanPath - scan a subquery-in-FROM
- ForeignPath - scan a foreign table, foreign join or foreign upper-relation
- CustomPath - for custom scan providers
- AppendPath - append multiple subpaths together
- MergeAppendPath - merge multiple subpaths, preserving their common sort order
- GroupResultPath - childless Result plan node (used for degenerate grouping)
- MaterialPath - a Material plan node
- MemoizePath - a Memoize plan node for caching tuples from sub-paths
- UniquePath - remove duplicate rows (either by hashing or sorting)
- GatherPath - collect the results of parallel workers
- GatherMergePath - collect parallel results, preserving their common sort order
- ProjectionPath - a Result plan node with child (used for projection)
- ProjectSetPath - a ProjectSet plan node applied to some sub-path
- SortPath - a Sort plan node applied to some sub-path
- IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
- GroupPath - a Group plan node applied to some sub-path
- UpperUniquePath - a Unique plan node applied to some sub-path
- AggPath - an Agg plan node applied to some sub-path
- GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
- MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
- WindowAggPath - a WindowAgg plan node applied to some sub-path
- SetOpPath - a SetOp plan node applied to some sub-path
- RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
- LockRowsPath - a LockRows plan node applied to some sub-path
- ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
- LimitPath - a Limit plan node applied to some sub-path
- NestPath - nested-loop joins
- MergePath - merge joins
- HashPath - hash joins
-
- EquivalenceClass - a data structure representing a set of values known equal
-
- PathKey - a data structure representing the sort ordering of a path
+ PlannerGlobal - global information for a single planner invocation
+
+ PlannerInfo - information for planning a particular Query (we make
+ a separate PlannerInfo node for each sub-Query)
+
+ RelOptInfo - a relation or joined relations
+
+ RestrictInfo - WHERE clauses, like "x = 3" or "y = z"
+ (note the same structure is used for restriction and
+ join clauses)
+
+ Path - every way to generate a RelOptInfo(sequential,index,joins)
+ A plain Path node can represent several simple plans, per its pathtype:
+ T_SeqScan - sequential scan
+ T_SampleScan - tablesample scan
+ T_FunctionScan - function-in-FROM scan
+ T_TableFuncScan - table function scan
+ T_ValuesScan - VALUES scan
+ T_CteScan - CTE (WITH) scan
+ T_NamedTuplestoreScan - ENR scan
+ T_WorkTableScan - scan worktable of a recursive CTE
+ T_Result - childless Result plan node (used for FROM-less SELECT)
+ IndexPath - index scan
+ BitmapHeapPath - top of a bitmapped index scan
+ TidPath - scan by CTID
+ TidRangePath - scan a contiguous range of CTIDs
+ SubqueryScanPath - scan a subquery-in-FROM
+ ForeignPath - scan a foreign table, foreign join or foreign upper-relation
+ CustomPath - for custom scan providers
+ AppendPath - append multiple subpaths together
+ MergeAppendPath - merge multiple subpaths, preserving their common sort order
+ GroupResultPath - childless Result plan node (used for degenerate grouping)
+ MaterialPath - a Material plan node
+ MemoizePath - a Memoize plan node for caching tuples from sub-paths
+ UniquePath - remove duplicate rows (either by hashing or sorting)
+ GatherPath - collect the results of parallel workers
+ GatherMergePath - collect parallel results, preserving their common sort order
+ ProjectionPath - a Result plan node with child (used for projection)
+ ProjectSetPath - a ProjectSet plan node applied to some sub-path
+ SortPath - a Sort plan node applied to some sub-path
+ IncrementalSortPath - an IncrementalSort plan node applied to some sub-path
+ GroupPath - a Group plan node applied to some sub-path
+ UpperUniquePath - a Unique plan node applied to some sub-path
+ AggPath - an Agg plan node applied to some sub-path
+ GroupingSetsPath - an Agg plan node used to implement GROUPING SETS
+ MinMaxAggPath - a Result plan node with subplans performing MIN/MAX
+ WindowAggPath - a WindowAgg plan node applied to some sub-path
+ SetOpPath - a SetOp plan node applied to some sub-path
+ RecursiveUnionPath - a RecursiveUnion plan node applied to two sub-paths
+ LockRowsPath - a LockRows plan node applied to some sub-path
+ ModifyTablePath - a ModifyTable plan node applied to some sub-path(s)
+ LimitPath - a Limit plan node applied to some sub-path
+ NestPath - nested-loop joins
+ MergePath - merge joins
+ HashPath - hash joins
+
+ EquivalenceClass - a data structure representing a set of values known equal
+
+ PathKey - a data structure representing the sort ordering of a path
The optimizer spends a good deal of its time worrying about the ordering
of the tuples returned by a path. The reason this is useful is that by
@@ -909,10 +921,10 @@ of the tuples generated by a particular Path. A path's pathkeys field is a
list of PathKey nodes, where the n'th item represents the n'th sort key of
the result. Each PathKey contains these fields:
- * a reference to an EquivalenceClass
- * a btree opfamily OID (must match one of those in the EC)
- * a sort direction (ascending or descending)
- * a nulls-first-or-last flag
+* a reference to an EquivalenceClass
+* a btree opfamily OID (must match one of those in the EC)
+* a sort direction (ascending or descending)
+* a nulls-first-or-last flag
The EquivalenceClass represents the value being sorted on. Since the
various members of an EquivalenceClass are known equal according to the
@@ -997,9 +1009,11 @@ lists (sort orderings) do not mention the same EquivalenceClass more than
once. For example, in all these cases the second sort column is redundant,
because it cannot distinguish values that are the same according to the
first sort column:
+
SELECT ... ORDER BY x, x
SELECT ... ORDER BY x, x DESC
SELECT ... WHERE x = y ORDER BY x, y
+
Although a user probably wouldn't write "ORDER BY x,x" directly, such
redundancies are more probable once equivalence classes have been
considered. Also, the system may generate redundant pathkey lists when
@@ -1350,14 +1364,14 @@ RelOptInfos are mostly dummy, but their pathlist lists hold all the Paths
considered useful for each step. Currently, we may create these types of
additional RelOptInfos during upper-level planning:
-UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
-UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
-UPPERREL_GROUP_AGG result of grouping/aggregation, if any
-UPPERREL_WINDOW result of window functions, if any
-UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
-UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
-UPPERREL_ORDERED result of ORDER BY, if any
-UPPERREL_FINAL result of any remaining top-level actions
+ UPPERREL_SETOP result of UNION/INTERSECT/EXCEPT, if any
+ UPPERREL_PARTIAL_GROUP_AGG result of partial grouping/aggregation, if any
+ UPPERREL_GROUP_AGG result of grouping/aggregation, if any
+ UPPERREL_WINDOW result of window functions, if any
+ UPPERREL_PARTIAL_DISTINCT result of partial "SELECT DISTINCT", if any
+ UPPERREL_DISTINCT result of "SELECT DISTINCT", if any
+ UPPERREL_ORDERED result of ORDER BY, if any
+ UPPERREL_FINAL result of any remaining top-level actions
UPPERREL_FINAL is used to represent any final processing steps, currently
LockRows (SELECT FOR UPDATE), LIMIT/OFFSET, and ModifyTable. There is no
diff --git a/src/backend/optimizer/plan/README b/src/backend/optimizer/plan/README
index 013c0f9ea2..04d9c50c51 100644
--- a/src/backend/optimizer/plan/README
+++ b/src/backend/optimizer/plan/README
@@ -6,32 +6,32 @@ Subselects
Vadim B. Mikheev
-From [email protected] Fri Feb 13 09:01:19 1998
-Received: from renoir.op.net ([email protected] [209.152.193.4])
- by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
- for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
-Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
-Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
-Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
-Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
-Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
-Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
- by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
- for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
- (envelope-from [email protected])
-Message-ID: <[email protected]>
-Date: Fri, 13 Feb 1998 20:50:50 +0700
-From: "Vadim B. Mikheev" <[email protected]>
-Organization: ITTS (Krasnoyarsk)
-X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
-MIME-Version: 1.0
-To: PostgreSQL Developers List <[email protected]>
-Subject: [HACKERS] Subselects are in CVS...
-Content-Type: text/plain; charset=us-ascii
-Content-Transfer-Encoding: 7bit
-Sender: [email protected]
-Precedence: bulk
-Status: OR
+ From [email protected] Fri Feb 13 09:01:19 1998
+ Received: from renoir.op.net ([email protected] [209.152.193.4])
+ by candle.pha.pa.us (8.8.5/8.8.5) with ESMTP id JAA11576
+ for <[email protected]>; Fri, 13 Feb 1998 09:01:17 -0500 (EST)
+ Received: from hub.org (hub.org [209.47.148.200]) by renoir.op.net (o1/$Revision: 1.14 $) with ESMTP id IAA09761 for <[email protected]>; Fri, 13 Feb 1998 08:41:22 -0500 (EST)
+ Received: from localhost (majordom@localhost) by hub.org (8.8.8/8.7.5) with SMTP id IAA08135; Fri, 13 Feb 1998 08:40:17 -0500 (EST)
+ Received: by hub.org (TLB v0.10a (1.23 tibbs 1997/01/09 00:29:32)); Fri, 13 Feb 1998 08:38:42 -0500 (EST)
+ Received: (from majordom@localhost) by hub.org (8.8.8/8.7.5) id IAA06646 for pgsql-hackers-outgoing; Fri, 13 Feb 1998 08:38:35 -0500 (EST)
+ Received: from dune.krasnet.ru (dune.krasnet.ru [193.125.44.86]) by hub.org (8.8.8/8.7.5) with ESMTP id IAA04568 for <[email protected]>; Fri, 13 Feb 1998 08:37:16 -0500 (EST)
+ Received: from sable.krasnoyarsk.su (dune.krasnet.ru [193.125.44.86])
+ by dune.krasnet.ru (8.8.7/8.8.7) with ESMTP id UAA13717
+ for <[email protected]>; Fri, 13 Feb 1998 20:51:03 +0700 (KRS)
+ (envelope-from [email protected])
+ Message-ID: <[email protected]>
+ Date: Fri, 13 Feb 1998 20:50:50 +0700
+ From: "Vadim B. Mikheev" <[email protected]>
+ Organization: ITTS (Krasnoyarsk)
+ X-Mailer: Mozilla 4.04 [en] (X11; I; FreeBSD 2.2.5-RELEASE i386)
+ MIME-Version: 1.0
+ To: PostgreSQL Developers List <[email protected]>
+ Subject: [HACKERS] Subselects are in CVS...
+ Content-Type: text/plain; charset=us-ascii
+ Content-Transfer-Encoding: 7bit
+ Sender: [email protected]
+ Precedence: bulk
+ Status: OR
This is some implementation notes and opened issues...
@@ -91,21 +91,21 @@ on each call) ExecReScan() now supports most of Plan types...
Explanation of EXPLAIN.
-vac=> explain select * from tmp where x >= (select max(x2) from test2
-where y2 = y and exists (select * from tempx where tx = x));
-NOTICE: QUERY PLAN:
+ vac=> explain select * from tmp where x >= (select max(x2) from test2
+ where y2 = y and exists (select * from tempx where tx = x));
+ NOTICE: QUERY PLAN:
-Seq Scan on tmp (cost=40.03 size=101 width=8)
- SubPlan
- ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
- -> Aggregate (cost=2.05 size=0 width=0)
- InitPlan
- ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
- -> Seq Scan on tempx (cost=4.33 size=1 width=4)
- -> Result (cost=2.05 size=0 width=0)
- ^^^^^^ EXISTS subsubquery was transformed into Param
- and so we have Result node here
- -> Index Scan on test2 (cost=2.05 size=1 width=4)
+ Seq Scan on tmp (cost=40.03 size=101 width=8)
+ SubPlan
+ ^^^^^^^ subquery is in Seq Scan' qual, its plan is below
+ -> Aggregate (cost=2.05 size=0 width=0)
+ InitPlan
+ ^^^^^^^^ EXISTS subsubquery is InitPlan of subquery
+ -> Seq Scan on tempx (cost=4.33 size=1 width=4)
+ -> Result (cost=2.05 size=0 width=0)
+ ^^^^^^ EXISTS subsubquery was transformed into Param
+ and so we have Result node here
+ -> Index Scan on test2 (cost=2.05 size=1 width=4)
Opened issues.
@@ -131,28 +131,28 @@ Results of some test. TMP is table with x,y (int4-s), x in 0-9,
y = 100 - x, 1000 tuples (10 duplicates of each tuple). TEST2 is table
with x2, y2 (int4-s), x2 in 1-99, y2 = 100 -x2, 10000 tuples (100 dups).
- Trying
+Trying
-select * from tmp where x >= (select max(x2) from test2 where y2 = y);
+ select * from tmp where x >= (select max(x2) from test2 where y2 = y);
- and
+and
-begin;
-select y as ty, max(x2) as mx into table tsub from test2, tmp
-where y2 = y group by ty;
-vacuum tsub;
-select x, y from tmp, tsub where x >= mx and y = ty;
-drop table tsub;
-end;
+ begin;
+ select y as ty, max(x2) as mx into table tsub from test2, tmp
+ where y2 = y group by ty;
+ vacuum tsub;
+ select x, y from tmp, tsub where x >= mx and y = ty;
+ drop table tsub;
+ end;
- Without index on test2(y2):
+Without index on test2(y2):
-SubSelect -> 320 sec
-Using temp table -> 32 sec
+ SubSelect -> 320 sec
+ Using temp table -> 32 sec
- Having index
+Having index
-SubSelect -> 17 sec (2M of memory)
-Using temp table -> 32 sec (12M of memory: -S 8192)
+ SubSelect -> 17 sec (2M of memory)
+ Using temp table -> 32 sec (12M of memory: -S 8192)
Vadim
diff --git a/src/backend/parser/README b/src/backend/parser/README
index e0c986a41e..9ac8026c73 100644
--- a/src/backend/parser/README
+++ b/src/backend/parser/README
@@ -7,27 +7,27 @@ This directory does more than tokenize and parse SQL queries. It also
creates Query structures for the various complex queries that are passed
to the optimizer and then executor.
-parser.c things start here
-scan.l break query into tokens
-scansup.c handle escapes in input strings
-gram.y parse the tokens and produce a "raw" parse tree
-analyze.c top level of parse analysis for optimizable queries
-parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
-parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
-parse_coerce.c handle coercing expressions to different data types
-parse_collate.c assign collation information in completed expressions
-parse_cte.c handle Common Table Expressions (WITH clauses)
-parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
-parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
-parse_func.c handle functions, table.column and column identifiers
-parse_merge.c handle MERGE
-parse_node.c create nodes for various structures
-parse_oper.c handle operators in expressions
-parse_param.c handle Params (for the cases used in the core backend)
-parse_relation.c support routines for tables and column handling
-parse_target.c handle the result list of the query
-parse_type.c support routines for data type handling
-parse_utilcmd.c parse analysis for utility commands (done at execution time)
+* parser.c things start here
+* scan.l break query into tokens
+* scansup.c handle escapes in input strings
+* gram.y parse the tokens and produce a "raw" parse tree
+* analyze.c top level of parse analysis for optimizable queries
+* parse_agg.c handle aggregates, like SUM(col1), AVG(col2), ...
+* parse_clause.c handle clauses like WHERE, ORDER BY, GROUP BY, ...
+* parse_coerce.c handle coercing expressions to different data types
+* parse_collate.c assign collation information in completed expressions
+* parse_cte.c handle Common Table Expressions (WITH clauses)
+* parse_expr.c handle expressions like col, col + 3, x = 3 or x = 4
+* parse_enr.c handle ephemeral named rels (trigger transition tables, ...)
+* parse_func.c handle functions, table.column and column identifiers
+* parse_merge.c handle MERGE
+* parse_node.c create nodes for various structures
+* parse_oper.c handle operators in expressions
+* parse_param.c handle Params (for the cases used in the core backend)
+* parse_relation.c support routines for tables and column handling
+* parse_target.c handle the result list of the query
+* parse_type.c support routines for data type handling
+* parse_utilcmd.c parse analysis for utility commands (done at execution time)
See also src/common/keywords.c, which contains the table of standard
keywords and the keyword lookup function. We separated that out because
diff --git a/src/backend/regex/README b/src/backend/regex/README
index 930d8ced0d..a0d45589b6 100644
--- a/src/backend/regex/README
+++ b/src/backend/regex/README
@@ -9,11 +9,13 @@ General source-file layout
There are six separately-compilable source files, five of which expose
exactly one exported function apiece:
- regcomp.c: pg_regcomp
- regexec.c: pg_regexec
- regerror.c: pg_regerror
- regfree.c: pg_regfree
- regprefix.c: pg_regprefix
+
+ regcomp.c pg_regcomp
+ regexec.c pg_regexec
+ regerror.c pg_regerror
+ regfree.c pg_regfree
+ regprefix.c pg_regprefix
+
(The pg_ prefixes were added by the Postgres project to distinguish this
library version from any similar one that might be present on a particular
system. They'd need to be removed or replaced in any standalone version
@@ -38,19 +40,19 @@ structs.)
What's where in src/backend/regex/:
-regcomp.c Top-level regex compilation code
-regc_color.c Color map management
-regc_cvec.c Character vector (cvec) management
-regc_lex.c Lexer
-regc_nfa.c NFA handling
-regc_locale.c Application-specific locale code from Tcl project
-regc_pg_locale.c Postgres-added application-specific locale code
-regexec.c Top-level regex execution code
-rege_dfa.c DFA creation and execution
-regerror.c pg_regerror: generate text for a regex error code
-regfree.c pg_regfree: API to free a no-longer-needed regex_t
-regexport.c Functions for extracting info from a regex_t
-regprefix.c Code for extracting a common prefix from a regex_t
+ regcomp.c Top-level regex compilation code
+ regc_color.c Color map management
+ regc_cvec.c Character vector (cvec) management
+ regc_lex.c Lexer
+ regc_nfa.c NFA handling
+ regc_locale.c Application-specific locale code from Tcl project
+ regc_pg_locale.c Postgres-added application-specific locale code
+ regexec.c Top-level regex execution code
+ rege_dfa.c DFA creation and execution
+ regerror.c pg_regerror: generate text for a regex error code
+ regfree.c pg_regfree: API to free a no-longer-needed regex_t
+ regexport.c Functions for extracting info from a regex_t
+ regprefix.c Code for extracting a common prefix from a regex_t
The locale-specific code is concerned primarily with case-folding and with
expanding locale-specific character classes, such as [[:alnum:]]. It
@@ -58,11 +60,11 @@ really needs refactoring if this is ever to become a standalone library.
The header files for the library are in src/include/regex/:
-regcustom.h Customizes library for particular application
-regerrs.h Error message list
-regex.h Exported API
-regexport.h Exported API for regexport.c
-regguts.h Internals declarations
+ regcustom.h Customizes library for particular application
+ regerrs.h Error message list
+ regex.h Exported API
+ regexport.h Exported API for regexport.c
+ regguts.h Internals declarations
DFAs, NFAs, and all that
@@ -261,15 +263,19 @@ an additional arc labeled 2 wherever there is an arc labeled 3; this
action ensures that characters of color 2 (i.e., "x") will still be
considered as allowing any transitions they did before. We are now done
parsing the regex, and we have these final color assignments:
+
color 1: "a"
color 2: "x"
color 3: other letters
color 4: digits
+
and the NFA has these arcs:
+
states 1 -> 2 on color 1 (hence, "a" only)
states 2 -> 3 on color 4 (digits)
states 3 -> 4 on colors 1, 3, 4, and 2 (covering all \w characters)
states 4 -> 5 on color 2 ("x" only)
+
which can be seen to be a correct representation of the regex.
There is one more complexity, which is how to handle ".", that is a
diff --git a/src/backend/snowball/README b/src/backend/snowball/README
index 675baff5c9..ab4e4044d3 100644
--- a/src/backend/snowball/README
+++ b/src/backend/snowball/README
@@ -10,12 +10,12 @@ which is released by them under a BSD-style license.
The Snowball project does not often make formal releases; it's best
to pull from their git repository
-git clone https://github.com/snowballstem/snowball.git
+ git clone https://github.com/snowballstem/snowball.git
and then building the derived files is as simple as
-cd snowball
-make
+ cd snowball
+ make
At least on Linux, no platform-specific adjustment is needed.
@@ -39,10 +39,10 @@ To update the PostgreSQL sources from a new Snowball version:
1. Copy the *.c files in snowball/src_c/ to src/backend/snowball/libstemmer
with replacement of "../runtime/header.h" by "header.h", for example
-for f in .../snowball/src_c/*.c
-do
- sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
-done
+ for f in .../snowball/src_c/*.c
+ do
+ sed 's|\.\./runtime/header\.h|header.h|' $f >libstemmer/`basename $f`
+ done
Do not copy stemmers that are listed in libstemmer/modules.txt as
nonstandard, such as "german2" or "lovins".
diff --git a/src/backend/storage/freespace/README b/src/backend/storage/freespace/README
index dc2a63a137..6d8882c3da 100644
--- a/src/backend/storage/freespace/README
+++ b/src/backend/storage/freespace/README
@@ -33,9 +33,9 @@ node stores the max amount of free space on any of its children.
For example:
- 4
- 4 2
-3 4 0 2 <- This level represents heap pages
+ 4
+ 4 2
+ 3 4 0 2 <- This level represents heap pages
We need two basic operations: search and update.
@@ -67,10 +67,10 @@ header takes some space on a page, the binary tree isn't perfect. That is,
a few right-most leaf nodes are missing, and there are some useless non-leaf
nodes at the right. So the tree looks something like this:
- 0
- 1 2
- 3 4 5 6
-7 8 9 A B
+ 0
+ 1 2
+ 3 4 5 6
+ 7 8 9 A B
where the numbers denote each node's position in the array. Note that the
tree is guaranteed complete above the leaf level; only some leaf nodes are
@@ -100,27 +100,27 @@ For example, assuming each FSM page can hold information about 4 pages (in
reality, it holds (BLCKSZ - headers) / 2, or ~4000 with default BLCKSZ),
we get a disk layout like this:
- 0 <-- page 0 at level 2 (root page)
- 0 <-- page 0 at level 1
- 0 <-- page 0 at level 0
- 1 <-- page 1 at level 0
- 2 <-- ...
- 3
- 1 <-- page 1 at level 1
- 4
- 5
- 6
- 7
- 2
- 8
- 9
- 10
- 11
- 3
- 12
- 13
- 14
- 15
+ 0 <-- page 0 at level 2 (root page)
+ 0 <-- page 0 at level 1
+ 0 <-- page 0 at level 0
+ 1 <-- page 1 at level 0
+ 2 <-- ...
+ 3
+ 1 <-- page 1 at level 1
+ 4
+ 5
+ 6
+ 7
+ 2
+ 8
+ 9
+ 10
+ 11
+ 3
+ 12
+ 13
+ 14
+ 15
where the numbers are page numbers *at that level*, starting from 0.
diff --git a/src/backend/storage/lmgr/README-SSI b/src/backend/storage/lmgr/README-SSI
index 50d2ecca9d..be9a6f96cf 100644
--- a/src/backend/storage/lmgr/README-SSI
+++ b/src/backend/storage/lmgr/README-SSI
@@ -199,55 +199,55 @@ The PostgreSQL implementation uses two additional optimizations:
PostgreSQL Implementation
-------------------------
- * Since this technique is based on Snapshot Isolation (SI), those
-areas in PostgreSQL which don't use SI can't be brought under SSI.
-This includes system tables, temporary tables, sequences, hint bit
-rewrites, etc. SSI can not eliminate existing anomalies in these
-areas.
-
- * Any transaction which is run at a transaction isolation level
-other than SERIALIZABLE will not be affected by SSI. If you want to
-enforce business rules through SSI, all transactions should be run at
-the SERIALIZABLE transaction isolation level, and that should
-probably be set as the default.
-
- * If all transactions are run at the SERIALIZABLE transaction
-isolation level, business rules can be enforced in triggers or
-application code without ever having a need to acquire an explicit
-lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
-
- * Those who want to continue to use snapshot isolation without
-the additional protections of SSI (and the associated costs of
-enforcing those protections), can use the REPEATABLE READ transaction
-isolation level. This level retains its legacy behavior, which
-is identical to the old SERIALIZABLE implementation and fully
-consistent with the standard's requirements for the REPEATABLE READ
-transaction isolation level.
-
- * Performance under this SSI implementation will be significantly
-improved if transactions which don't modify permanent tables are
-declared to be READ ONLY before they begin reading data.
-
- * Performance under SSI will tend to degrade more rapidly with a
-large number of active database transactions than under less strict
-isolation levels. Limiting the number of active transactions through
-use of a connection pool or similar techniques may be necessary to
-maintain good performance.
-
- * Any transaction which must be rolled back to prevent
-serialization anomalies will fail with SQLSTATE 40001, which has a
-standard meaning of "serialization failure".
-
- * This SSI implementation makes an effort to choose the
-transaction to be canceled such that an immediate retry of the
-transaction will not fail due to conflicts with exactly the same
-transactions. Pursuant to this goal, no transaction is canceled
-until one of the other transactions in the set of conflicts which
-could generate an anomaly has successfully committed. This is
-conceptually similar to how write conflicts are handled. To fully
-implement this guarantee there needs to be a way to roll back the
-active transaction for another process with a serialization failure
-SQLSTATE, even if it is "idle in transaction".
+* Since this technique is based on Snapshot Isolation (SI), those
+ areas in PostgreSQL which don't use SI can't be brought under SSI.
+ This includes system tables, temporary tables, sequences, hint bit
+ rewrites, etc. SSI can not eliminate existing anomalies in these
+ areas.
+
+* Any transaction which is run at a transaction isolation level
+ other than SERIALIZABLE will not be affected by SSI. If you want to
+ enforce business rules through SSI, all transactions should be run at
+ the SERIALIZABLE transaction isolation level, and that should
+ probably be set as the default.
+
+* If all transactions are run at the SERIALIZABLE transaction
+ isolation level, business rules can be enforced in triggers or
+ application code without ever having a need to acquire an explicit
+ lock or to use SELECT FOR SHARE or SELECT FOR UPDATE.
+
+* Those who want to continue to use snapshot isolation without
+ the additional protections of SSI (and the associated costs of
+ enforcing those protections), can use the REPEATABLE READ transaction
+ isolation level. This level retains its legacy behavior, which
+ is identical to the old SERIALIZABLE implementation and fully
+ consistent with the standard's requirements for the REPEATABLE READ
+ transaction isolation level.
+
+* Performance under this SSI implementation will be significantly
+ improved if transactions which don't modify permanent tables are
+ declared to be READ ONLY before they begin reading data.
+
+* Performance under SSI will tend to degrade more rapidly with a
+ large number of active database transactions than under less strict
+ isolation levels. Limiting the number of active transactions through
+ use of a connection pool or similar techniques may be necessary to
+ maintain good performance.
+
+* Any transaction which must be rolled back to prevent
+ serialization anomalies will fail with SQLSTATE 40001, which has a
+ standard meaning of "serialization failure".
+
+* This SSI implementation makes an effort to choose the
+ transaction to be canceled such that an immediate retry of the
+ transaction will not fail due to conflicts with exactly the same
+ transactions. Pursuant to this goal, no transaction is canceled
+ until one of the other transactions in the set of conflicts which
+ could generate an anomaly has successfully committed. This is
+ conceptually similar to how write conflicts are handled. To fully
+ implement this guarantee there needs to be a way to roll back the
+ active transaction for another process with a serialization failure
+ SQLSTATE, even if it is "idle in transaction".
Predicate Locking
@@ -303,23 +303,23 @@ Heap locking
Predicate locks will be acquired for the heap based on the following:
- * For a table scan, the entire relation will be locked.
+* For a table scan, the entire relation will be locked.
- * Each tuple read which is visible to the reading transaction
-will be locked, whether or not it meets selection criteria; except
-that there is no need to acquire an SIREAD lock on a tuple when the
-transaction already holds a write lock on any tuple representing the
-row, since a rw-conflict would also create a ww-dependency which
-has more aggressive enforcement and thus will prevent any anomaly.
+* Each tuple read which is visible to the reading transaction
+ will be locked, whether or not it meets selection criteria; except
+ that there is no need to acquire an SIREAD lock on a tuple when the
+ transaction already holds a write lock on any tuple representing the
+ row, since a rw-conflict would also create a ww-dependency which
+ has more aggressive enforcement and thus will prevent any anomaly.
- * Modifying a heap tuple creates a rw-conflict with any transaction
-that holds a SIREAD lock on that tuple, or on the page or relation
-that contains it.
+* Modifying a heap tuple creates a rw-conflict with any transaction
+ that holds a SIREAD lock on that tuple, or on the page or relation
+ that contains it.
- * Inserting a new tuple creates a rw-conflict with any transaction
-holding a SIREAD lock on the entire relation. It doesn't conflict with
-page-level locks, because page-level locks are only used to aggregate
-tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
+* Inserting a new tuple creates a rw-conflict with any transaction
+ holding a SIREAD lock on the entire relation. It doesn't conflict with
+ page-level locks, because page-level locks are only used to aggregate
+ tuple locks. Unlike index page locks, they don't lock "gaps" on the page.
Index AM implementations
@@ -346,60 +346,60 @@ false positives, they should be minimized for performance reasons.
Several optimizations are possible, though not all are implemented yet:
- * An index scan which is just finding the right position for an
-index insertion or deletion need not acquire a predicate lock.
+* An index scan which is just finding the right position for an
+ index insertion or deletion need not acquire a predicate lock.
- * An index scan which is comparing for equality on the entire key
-for a unique index need not acquire a predicate lock as long as a key
-is found corresponding to a visible tuple which has not been modified
-by another transaction -- there are no "between or around" gaps to
-cover.
+* An index scan which is comparing for equality on the entire key
+ for a unique index need not acquire a predicate lock as long as a key
+ is found corresponding to a visible tuple which has not been modified
+ by another transaction -- there are no "between or around" gaps to
+ cover.
- * As long as built-in foreign key enforcement continues to use
-its current "special tricks" to deal with MVCC issues, predicate
-locks should not be needed for scans done by enforcement code.
+* As long as built-in foreign key enforcement continues to use
+ its current "special tricks" to deal with MVCC issues, predicate
+ locks should not be needed for scans done by enforcement code.
- * If a search determines that no rows can be found regardless of
-index contents because the search conditions are contradictory (e.g.,
-x = 1 AND x = 2), then no predicate lock is needed.
+* If a search determines that no rows can be found regardless of
+ index contents because the search conditions are contradictory (e.g.,
+ x = 1 AND x = 2), then no predicate lock is needed.
Other index AM implementation considerations:
- * For an index AM that doesn't have support for predicate locking,
-we just acquire a predicate lock on the whole index for any search.
-
- * B-tree index searches acquire predicate locks only on the
-index *leaf* pages needed to lock the appropriate index range. If,
-however, a search discovers that no root page has yet been created, a
-predicate lock on the index relation is required.
-
- * Like a B-tree, GIN searches acquire predicate locks only on the
-leaf pages of entry tree. When performing an equality scan, and an
-entry has a posting tree, the posting tree root is locked instead, to
-lock only that key value. However, fastupdate=on postpones the
-insertion of tuples into index structure by temporarily storing them
-into pending list. That makes us unable to detect r-w conflicts using
-page-level locks. To cope with that, insertions to the pending list
-conflict with all scans.
-
- * GiST searches can determine that there are no matches at any
-level of the index, so we acquire predicate lock at each index
-level during a GiST search. An index insert at the leaf level can
-then be trusted to ripple up to all levels and locations where
-conflicting predicate locks may exist. In case there is a page split,
-we need to copy predicate lock from the original page to all the new
-pages.
-
- * Hash index searches acquire predicate locks on the primary
-page of a bucket. It acquires a lock on both the old and new buckets
-for scans that happen concurrently with page splits. During a bucket
-split, a predicate lock is copied from the primary page of an old
-bucket to the primary page of a new bucket.
-
- * The effects of page splits, overflows, consolidations, and
-removals must be carefully reviewed to ensure that predicate locks
-aren't "lost" during those operations, or kept with pages which could
-get re-used for different parts of the index.
+* For an index AM that doesn't have support for predicate locking,
+ we just acquire a predicate lock on the whole index for any search.
+
+* B-tree index searches acquire predicate locks only on the
+ index *leaf* pages needed to lock the appropriate index range. If,
+ however, a search discovers that no root page has yet been created, a
+ predicate lock on the index relation is required.
+
+* Like a B-tree, GIN searches acquire predicate locks only on the
+ leaf pages of entry tree. When performing an equality scan, and an
+ entry has a posting tree, the posting tree root is locked instead, to
+ lock only that key value. However, fastupdate=on postpones the
+ insertion of tuples into index structure by temporarily storing them
+ into pending list. That makes us unable to detect r-w conflicts using
+ page-level locks. To cope with that, insertions to the pending list
+ conflict with all scans.
+
+* GiST searches can determine that there are no matches at any
+ level of the index, so we acquire predicate lock at each index
+ level during a GiST search. An index insert at the leaf level can
+ then be trusted to ripple up to all levels and locations where
+ conflicting predicate locks may exist. In case there is a page split,
+ we need to copy predicate lock from the original page to all the new
+ pages.
+
+* Hash index searches acquire predicate locks on the primary
+ page of a bucket. It acquires a lock on both the old and new buckets
+ for scans that happen concurrently with page splits. During a bucket
+ split, a predicate lock is copied from the primary page of an old
+ bucket to the primary page of a new bucket.
+
+* The effects of page splits, overflows, consolidations, and
+ removals must be carefully reviewed to ensure that predicate locks
+ aren't "lost" during those operations, or kept with pages which could
+ get re-used for different parts of the index.
Innovations
@@ -409,183 +409,183 @@ The PostgreSQL implementation of Serializable Snapshot Isolation
differs from what is described in the cited papers for several
reasons:
- 1. PostgreSQL didn't have any existing predicate locking. It had
+1. PostgreSQL didn't have any existing predicate locking. It had
to be added from scratch.
- 2. The existing in-memory lock structures were not suitable for
+2. The existing in-memory lock structures were not suitable for
tracking SIREAD locks.
- * In PostgreSQL, tuple level locks are not held in RAM for
+ - In PostgreSQL, tuple level locks are not held in RAM for
any length of time; lock information is written to the tuples
involved in the transactions.
- * In PostgreSQL, existing lock structures have pointers to
+ - In PostgreSQL, existing lock structures have pointers to
memory which is related to a session. SIREAD locks need to persist
past the end of the originating transaction and even the session
which ran it.
- * PostgreSQL needs to be able to tolerate a large number of
+ - PostgreSQL needs to be able to tolerate a large number of
transactions executing while one long-running transaction stays open
-- the in-RAM techniques discussed in the papers wouldn't support
that.
- 3. Unlike the database products used for the prototypes described
+3. Unlike the database products used for the prototypes described
in the papers, PostgreSQL didn't already have a true serializable
isolation level distinct from snapshot isolation.
- 4. PostgreSQL supports subtransactions -- an issue not mentioned
+4. PostgreSQL supports subtransactions -- an issue not mentioned
in the papers.
- 5. PostgreSQL doesn't assign a transaction number to a database
+5. PostgreSQL doesn't assign a transaction number to a database
transaction until and unless necessary (normally, when the transaction
attempts to modify data).
- 6. PostgreSQL has pluggable data types with user-definable
+6. PostgreSQL has pluggable data types with user-definable
operators, as well as pluggable index types, not all of which are
based around data types which support ordering.
- 7. Some possible optimizations became apparent during development
+7. Some possible optimizations became apparent during development
and testing.
Differences from the implementation described in the papers are
listed below.
- * New structures needed to be created in shared memory to track
-the proper information for serializable transactions and their SIREAD
-locks.
-
- * Because PostgreSQL does not have the same concept of an "oldest
-transaction ID" for all serializable transactions as assumed in the
-Cahill thesis, we track the oldest snapshot xmin among serializable
-transactions, and a count of how many active transactions use that
-xmin. When the count hits zero we find the new oldest xmin and run a
-clean-up based on that.
-
- * Because reads in a subtransaction may cause that subtransaction
-to roll back, thereby affecting what is written by the top level
-transaction, predicate locks must survive a subtransaction rollback.
-As a consequence, all xid usage in SSI, including predicate locking,
-is based on the top level xid. When looking at an xid that comes
-from a tuple's xmin or xmax, for example, we always call
-SubTransGetTopmostTransaction() before doing much else with it.
-
- * PostgreSQL does not use "update in place" with a rollback log
-for its MVCC implementation. Where possible it uses "HOT" updates on
-the same page (if there is room and no indexed value is changed).
-For non-HOT updates the old tuple is expired in place and a new tuple
-is inserted at a new location. Because of this difference, a tuple
-lock in PostgreSQL doesn't automatically lock any other versions of a
-row. We don't try to copy or expand a tuple lock to any other
-versions of the row, based on the following proof that any additional
-serialization failures we would get from that would be false
-positives:
-
- o If transaction T1 reads a row version (thus acquiring a
-predicate lock on it) and a second transaction T2 updates that row
-version (thus creating a rw-conflict graph edge from T1 to T2), must a
-third transaction T3 which re-updates the new version of the row also
-have a rw-conflict in from T1 to prevent anomalies? In other words,
-does it matter whether we recognize the edge T1 -> T3?
-
- o If T1 has a conflict in, it certainly doesn't. Adding the
-edge T1 -> T3 would create a dangerous structure, but we already had
-one from the edge T1 -> T2, so we would have aborted something anyway.
-(T2 has already committed, else T3 could not have updated its output;
-but we would have aborted either T1 or T1's predecessor(s). Hence
-no cycle involving T1 and T3 can survive.)
-
- o Now let's consider the case where T1 doesn't have a
-rw-conflict in. If that's the case, for this edge T1 -> T3 to make a
-difference, T3 must have a rw-conflict out that induces a cycle in the
-dependency graph, i.e. a conflict out to some transaction preceding T1
-in the graph. (A conflict out to T1 itself would be problematic too,
-but that would mean T1 has a conflict in, the case we already
-eliminated.)
-
- o So now we're trying to figure out if there can be an
-rw-conflict edge T3 -> T0, where T0 is some transaction that precedes
-T1. For T0 to precede T1, there has to be some edge, or sequence of
-edges, from T0 to T1. At least the last edge has to be a wr-dependency
-or ww-dependency rather than a rw-conflict, because T1 doesn't have a
-rw-conflict in. And that gives us enough information about the order
-of transactions to see that T3 can't have a rw-conflict to T0:
- - T0 committed before T1 started (the wr/ww-dependency implies this)
- - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
- - T2 committed before T3 started (otherwise, T3 would get aborted
+* New structures needed to be created in shared memory to track
+ the proper information for serializable transactions and their SIREAD
+ locks.
+
+* Because PostgreSQL does not have the same concept of an "oldest
+ transaction ID" for all serializable transactions as assumed in the
+ Cahill thesis, we track the oldest snapshot xmin among serializable
+ transactions, and a count of how many active transactions use that
+ xmin. When the count hits zero we find the new oldest xmin and run a
+ clean-up based on that.
+
+* Because reads in a subtransaction may cause that subtransaction
+ to roll back, thereby affecting what is written by the top level
+ transaction, predicate locks must survive a subtransaction rollback.
+ As a consequence, all xid usage in SSI, including predicate locking,
+ is based on the top level xid. When looking at an xid that comes
+ from a tuple's xmin or xmax, for example, we always call
+ SubTransGetTopmostTransaction() before doing much else with it.
+
+* PostgreSQL does not use "update in place" with a rollback log
+ for its MVCC implementation. Where possible it uses "HOT" updates on
+ the same page (if there is room and no indexed value is changed).
+ For non-HOT updates the old tuple is expired in place and a new tuple
+ is inserted at a new location. Because of this difference, a tuple
+ lock in PostgreSQL doesn't automatically lock any other versions of a
+ row. We don't try to copy or expand a tuple lock to any other
+ versions of the row, based on the following proof that any additional
+ serialization failures we would get from that would be false
+ positives:
+
+ - If transaction T1 reads a row version (thus acquiring a
+ predicate lock on it) and a second transaction T2 updates that row
+ version (thus creating a rw-conflict graph edge from T1 to T2), must a
+ third transaction T3 which re-updates the new version of the row also
+ have a rw-conflict in from T1 to prevent anomalies? In other words,
+ does it matter whether we recognize the edge T1 -> T3?
+
+ - If T1 has a conflict in, it certainly doesn't. Adding the
+ edge T1 -> T3 would create a dangerous structure, but we already had
+ one from the edge T1 -> T2, so we would have aborted something anyway.
+ (T2 has already committed, else T3 could not have updated its output;
+ but we would have aborted either T1 or T1's predecessor(s). Hence
+ no cycle involving T1 and T3 can survive.)
+
+ - Now let's consider the case where T1 doesn't have a
+ rw-conflict in. If that's the case, for this edge T1 -> T3 to make a
+ difference, T3 must have a rw-conflict out that induces a cycle in the
+ dependency graph, i.e. a conflict out to some transaction preceding T1
+ in the graph. (A conflict out to T1 itself would be problematic too,
+ but that would mean T1 has a conflict in, the case we already
+ eliminated.)
+
+ - So now we're trying to figure out if there can be an
+ rw-conflict edge T3 -> T0, where T0 is some transaction that precedes
+ T1. For T0 to precede T1, there has to be some edge, or sequence of
+ edges, from T0 to T1. At least the last edge has to be a wr-dependency
+ or ww-dependency rather than a rw-conflict, because T1 doesn't have a
+ rw-conflict in. And that gives us enough information about the order
+ of transactions to see that T3 can't have a rw-conflict to T0:
+ - T0 committed before T1 started (the wr/ww-dependency implies this)
+ - T1 started before T2 committed (the T1->T2 rw-conflict implies this)
+ - T2 committed before T3 started (otherwise, T3 would get aborted
because of an update conflict)
- o That means T0 committed before T3 started, and therefore
-there can't be a rw-conflict from T3 to T0.
+ - That means T0 committed before T3 started, and therefore
+ there can't be a rw-conflict from T3 to T0.
- o So in all cases, we don't need the T1 -> T3 edge to
-recognize cycles. Therefore it's not necessary for T1's SIREAD lock
-on the original tuple version to cover later versions as well.
+ - So in all cases, we don't need the T1 -> T3 edge to
+ recognize cycles. Therefore it's not necessary for T1's SIREAD lock
+ on the original tuple version to cover later versions as well.
- * Predicate locking in PostgreSQL starts at the tuple level
-when possible. Multiple fine-grained locks are promoted to a single
-coarser-granularity lock as needed to avoid resource exhaustion. The
-amount of memory used for these structures is configurable, to balance
-RAM usage against SIREAD lock granularity.
+* Predicate locking in PostgreSQL starts at the tuple level
+ when possible. Multiple fine-grained locks are promoted to a single
+ coarser-granularity lock as needed to avoid resource exhaustion. The
+ amount of memory used for these structures is configurable, to balance
+ RAM usage against SIREAD lock granularity.
- * Each backend keeps a process-local table of the locks it holds.
-To support granularity promotion decisions with low CPU and locking
-overhead, this table also includes the coarser covering locks and the
-number of finer-granularity locks they cover.
+* Each backend keeps a process-local table of the locks it holds.
+ To support granularity promotion decisions with low CPU and locking
+ overhead, this table also includes the coarser covering locks and the
+ number of finer-granularity locks they cover.
- * Conflicts are identified by looking for predicate locks
-when tuples are written, and by looking at the MVCC information when
-tuples are read. There is no matching between two RAM-based locks.
+* Conflicts are identified by looking for predicate locks
+ when tuples are written, and by looking at the MVCC information when
+ tuples are read. There is no matching between two RAM-based locks.
- * Because write locks are stored in the heap tuples rather than a
-RAM-based lock table, the optimization described in the Cahill thesis
-which eliminates an SIREAD lock where there is a write lock is
-implemented by the following:
+* Because write locks are stored in the heap tuples rather than a
+ RAM-based lock table, the optimization described in the Cahill thesis
+ which eliminates an SIREAD lock where there is a write lock is
+ implemented by the following:
1. When checking a heap write for conflicts against existing
-predicate locks, a tuple lock on the tuple being written is removed.
+ predicate locks, a tuple lock on the tuple being written is removed.
2. When acquiring a predicate lock on a heap tuple, we
-return quickly without doing anything if it is a tuple written by the
-reading transaction.
-
- * Rather than using conflictIn and conflictOut pointers which use
-NULL to indicate no conflict and a self-reference to indicate
-multiple conflicts or conflicts with committed transactions, we use a
-list of rw-conflicts. With the more complete information, false
-positives are reduced and we have sufficient data for more aggressive
-clean-up and other optimizations:
-
- o We can avoid ever rolling back a transaction until and
-unless there is a pivot where a transaction on the conflict *out*
-side of the pivot committed before either of the other transactions.
-
- o We can avoid ever rolling back a transaction when the
-transaction on the conflict *in* side of the pivot is explicitly or
-implicitly READ ONLY unless the transaction on the conflict *out*
-side of the pivot committed before the READ ONLY transaction acquired
-its snapshot. (An implicit READ ONLY transaction is one which
-committed without writing, even though it was not explicitly declared
-to be READ ONLY.)
-
- o We can more aggressively clean up conflicts, predicate
-locks, and SSI transaction information.
-
- * We allow a READ ONLY transaction to "opt out" of SSI if there are
-no READ WRITE transactions which could cause the READ ONLY
-transaction to ever become part of a "dangerous structure" of
-overlapping transaction dependencies.
-
- * We allow the user to request that a READ ONLY transaction wait
-until the conditions are right for it to start in the "opt out" state
-described above. We add a DEFERRABLE state to transactions, which is
-specified and maintained in a way similar to READ ONLY. It is
-ignored for transactions that are not SERIALIZABLE and READ ONLY.
-
- * When a transaction must be rolled back, we pick among the
-active transactions such that an immediate retry will not fail again
-on conflicts with the same transactions.
-
- * We use the PostgreSQL SLRU system to hold summarized
-information about older committed transactions to put an upper bound
-on RAM used. Beyond that limit, information spills to disk.
-Performance can degrade in a pessimal situation, but it should be
-tolerable, and transactions won't need to be canceled or blocked
-from starting.
+ return quickly without doing anything if it is a tuple written by the
+ reading transaction.
+
+* Rather than using conflictIn and conflictOut pointers which use
+ NULL to indicate no conflict and a self-reference to indicate
+ multiple conflicts or conflicts with committed transactions, we use a
+ list of rw-conflicts. With the more complete information, false
+ positives are reduced and we have sufficient data for more aggressive
+ clean-up and other optimizations:
+
+ - We can avoid ever rolling back a transaction until and
+ unless there is a pivot where a transaction on the conflict *out*
+ side of the pivot committed before either of the other transactions.
+
+ - We can avoid ever rolling back a transaction when the
+ transaction on the conflict *in* side of the pivot is explicitly or
+ implicitly READ ONLY unless the transaction on the conflict *out*
+ side of the pivot committed before the READ ONLY transaction acquired
+ its snapshot. (An implicit READ ONLY transaction is one which
+ committed without writing, even though it was not explicitly declared
+ to be READ ONLY.)
+
+ - We can more aggressively clean up conflicts, predicate
+ locks, and SSI transaction information.
+
+* We allow a READ ONLY transaction to "opt out" of SSI if there are
+ no READ WRITE transactions which could cause the READ ONLY
+ transaction to ever become part of a "dangerous structure" of
+ overlapping transaction dependencies.
+
+* We allow the user to request that a READ ONLY transaction wait
+ until the conditions are right for it to start in the "opt out" state
+ described above. We add a DEFERRABLE state to transactions, which is
+ specified and maintained in a way similar to READ ONLY. It is
+ ignored for transactions that are not SERIALIZABLE and READ ONLY.
+
+* When a transaction must be rolled back, we pick among the
+ active transactions such that an immediate retry will not fail again
+ on conflicts with the same transactions.
+
+* We use the PostgreSQL SLRU system to hold summarized
+ information about older committed transactions to put an upper bound
+ on RAM used. Beyond that limit, information spills to disk.
+ Performance can degrade in a pessimal situation, but it should be
+ tolerable, and transactions won't need to be canceled or blocked
+ from starting.
R&D Issues
@@ -594,34 +594,34 @@ R&D Issues
This is intended to be the place to record specific issues which need
more detailed review or analysis.
- * WAL file replay. While serializable implementations using S2PL
-can guarantee that the write-ahead log contains commits in a sequence
-consistent with some serial execution of serializable transactions,
-SSI cannot make that guarantee. While the WAL replay is no less
-consistent than under snapshot isolation, it is possible that under
-PITR recovery or hot standby a database could reach a readable state
-where some transactions appear before other transactions which would
-have had to precede them to maintain serializable consistency. In
-essence, if we do nothing, WAL replay will be at snapshot isolation
-even for serializable transactions. Is this OK? If not, how do we
-address it?
-
- * External replication. Look at how this impacts external
-replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
-This is related to the "WAL file replay" issue.
-
- * UNIQUE btree search for equality on all columns. Since a search
-of a UNIQUE index using equality tests on all columns will lock the
-heap tuple if an entry is found, it appears that there is no need to
-get a predicate lock on the index in that case. A predicate lock is
-still needed for such a search if a matching index entry which points
-to a visible tuple is not found.
-
- * Minimize touching of shared memory. Should lists in shared
-memory push entries which have just been returned to the front of the
-available list, so they will be popped back off soon and some memory
-might never be touched, or should we keep adding returned items to
-the end of the available list?
+* WAL file replay. While serializable implementations using S2PL
+ can guarantee that the write-ahead log contains commits in a sequence
+ consistent with some serial execution of serializable transactions,
+ SSI cannot make that guarantee. While the WAL replay is no less
+ consistent than under snapshot isolation, it is possible that under
+ PITR recovery or hot standby a database could reach a readable state
+ where some transactions appear before other transactions which would
+ have had to precede them to maintain serializable consistency. In
+ essence, if we do nothing, WAL replay will be at snapshot isolation
+ even for serializable transactions. Is this OK? If not, how do we
+ address it?
+
+* External replication. Look at how this impacts external
+ replication solutions, like Postgres-R, Slony, pgpool, HS/SR, etc.
+ This is related to the "WAL file replay" issue.
+
+* UNIQUE btree search for equality on all columns. Since a search
+ of a UNIQUE index using equality tests on all columns will lock the
+ heap tuple if an entry is found, it appears that there is no need to
+ get a predicate lock on the index in that case. A predicate lock is
+ still needed for such a search if a matching index entry which points
+ to a visible tuple is not found.
+
+* Minimize touching of shared memory. Should lists in shared
+ memory push entries which have just been returned to the front of the
+ available list, so they will be popped back off soon and some memory
+ might never be touched, or should we keep adding returned items to
+ the end of the available list?
References
@@ -638,9 +638,9 @@ http://dx.doi.org/10.1145/1071610.1071615
Architecture of a Database System. Foundations and Trends(R) in
Databases Vol. 1, No. 2 (2007) 141-259.
http://db.cs.berkeley.edu/papers/fntdb07-architecture.pdf
- Of particular interest:
- * 6.1 A Note on ACID
- * 6.2 A Brief Review of Serializability
- * 6.3 Locking and Latching
- * 6.3.1 Transaction Isolation Levels
- * 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
+Of particular interest:
+* 6.1 A Note on ACID
+* 6.2 A Brief Review of Serializability
+* 6.3 Locking and Latching
+* 6.3.1 Transaction Isolation Levels
+* 6.5.3 Next-Key Locking: Physical Surrogates for Logical Properties
diff --git a/src/backend/utils/fmgr/README b/src/backend/utils/fmgr/README
index 9958d38992..3edfbe933d 100644
--- a/src/backend/utils/fmgr/README
+++ b/src/backend/utils/fmgr/README
@@ -20,18 +20,18 @@ tuple.)
When a function is looked up in pg_proc, the result is represented as
-typedef struct
-{
- PGFunction fn_addr; /* pointer to function or handler to be called */
- Oid fn_oid; /* OID of function (NOT of handler, if any) */
- short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
- bool fn_strict; /* function is "strict" (NULL in => NULL out) */
- bool fn_retset; /* function returns a set (over multiple calls) */
- unsigned char fn_stats; /* collect stats if track_functions > this */
- void *fn_extra; /* extra space for use by handler */
- MemoryContext fn_mcxt; /* memory context to store fn_extra in */
- Node *fn_expr; /* expression parse tree for call, or NULL */
-} FmgrInfo;
+ typedef struct
+ {
+ PGFunction fn_addr; /* pointer to function or handler to be called */
+ Oid fn_oid; /* OID of function (NOT of handler, if any) */
+ short fn_nargs; /* number of input args (0..FUNC_MAX_ARGS) */
+ bool fn_strict; /* function is "strict" (NULL in => NULL out) */
+ bool fn_retset; /* function returns a set (over multiple calls) */
+ unsigned char fn_stats; /* collect stats if track_functions > this */
+ void *fn_extra; /* extra space for use by handler */
+ MemoryContext fn_mcxt; /* memory context to store fn_extra in */
+ Node *fn_expr; /* expression parse tree for call, or NULL */
+ } FmgrInfo;
For an ordinary built-in function, fn_addr is just the address of the C
routine that implements the function. Otherwise it is the address of a
@@ -59,17 +59,17 @@ FmgrInfo than in FunctionCallInfoBaseData where it might more logically go.
During a call of a function, the following data structure is created
and passed to the function:
-typedef struct
-{
- FmgrInfo *flinfo; /* ptr to lookup info used for this call */
- Node *context; /* pass info about context of call */
- Node *resultinfo; /* pass or return extra info about result */
- Oid fncollation; /* collation for function to use */
- bool isnull; /* function must set true if result is NULL */
- short nargs; /* # arguments actually passed */
- NullableDatum args[]; /* Arguments passed to function */
-} FunctionCallInfoBaseData;
-typedef FunctionCallInfoBaseData* FunctionCallInfo;
+ typedef struct
+ {
+ FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+ Node *context; /* pass info about context of call */
+ Node *resultinfo; /* pass or return extra info about result */
+ Oid fncollation; /* collation for function to use */
+ bool isnull; /* function must set true if result is NULL */
+ short nargs; /* # arguments actually passed */
+ NullableDatum args[]; /* Arguments passed to function */
+ } FunctionCallInfoBaseData;
+ typedef FunctionCallInfoBaseData* FunctionCallInfo;
flinfo points to the lookup info used to make the call. Ordinary functions
will probably ignore this field, but function class handlers will need it
@@ -119,11 +119,11 @@ least), and other uglinesses.
Callees, whether they be individual functions or function handlers,
shall always have this signature:
-Datum function (FunctionCallInfo fcinfo);
+ Datum function (FunctionCallInfo fcinfo);
which is represented by the typedef
-typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
+ typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
The function is responsible for setting fcinfo->isnull appropriately
as well as returning a result represented as a Datum. Note that since
@@ -139,11 +139,11 @@ Here are the proposed macros and coding conventions:
The definition of an fmgr-callable function will always look like
-Datum
-function_name(PG_FUNCTION_ARGS)
-{
- ...
-}
+ Datum
+ function_name(PG_FUNCTION_ARGS)
+ {
+ ...
+ }
"PG_FUNCTION_ARGS" just expands to "FunctionCallInfo fcinfo". The main
reason for using this macro is to make it easy for scripts to spot function
@@ -156,26 +156,36 @@ just "fcinfo->args[n].isnull"). It should avoid trying to fetch the value
of any argument that is null.
Both strict and nonstrict functions can return NULL, if needed, with
+
PG_RETURN_NULL();
+
which expands to
+
{ fcinfo->isnull = true; return (Datum) 0; }
Argument values are ordinarily fetched using code like
+
int32 name = PG_GETARG_INT32(number);
For float4, float8, and int8, the PG_GETARG macros will hide whether the
types are pass-by-value or pass-by-reference. For example, if float8 is
pass-by-reference then PG_GETARG_FLOAT8 expands to
+
(* (float8 *) DatumGetPointer(fcinfo->args[number].value))
+
and would typically be called like this:
+
float8 arg = PG_GETARG_FLOAT8(0);
+
For what are now historical reasons, the float-related typedefs and macros
express the type width in bytes (4 or 8), whereas we prefer to label the
widths of integer types in bits.
Non-null values are returned with a PG_RETURN_XXX macro of the appropriate
type. For example, PG_RETURN_INT32 expands to
+
return Int32GetDatum(x)
+
PG_RETURN_FLOAT4, PG_RETURN_FLOAT8, and PG_RETURN_INT64 hide whether their
data types are pass-by-value or pass-by-reference, by doing a palloc if
needed.
@@ -290,9 +300,13 @@ transaction cleanup. SQL-callable functions can support this need
using the ErrorSaveContext context mechanism.
To report a "soft" error, a SQL-callable function should call
+
errsave(fcinfo->context, ...)
+
where it would previously have done
+
ereport(ERROR, ...)
+
If the passed "context" is NULL or is not an ErrorSaveContext node,
then errsave behaves precisely as ereport(ERROR): the exception is
thrown via longjmp, so that control does not return. If "context"
@@ -306,7 +320,9 @@ been reported in the ErrorSaveContext node.)
If there is nothing to do except return after calling errsave(),
you can save a line or two by writing
+
ereturn(fcinfo->context, dummy_value, ...)
+
to perform errsave() and then "return dummy_value".
An error reported "softly" must be safe, in the sense that there is
diff --git a/src/backend/utils/mb/README b/src/backend/utils/mb/README
index 4447881ead..afbe4c65d1 100644
--- a/src/backend/utils/mb/README
+++ b/src/backend/utils/mb/README
@@ -3,17 +3,17 @@ src/backend/utils/mb/README
Encodings
=========
-conv.c: static functions and a public table for code conversion
-mbutils.c: public functions for the backend only.
-stringinfo_mb.c: public backend-only multibyte-aware stringinfo functions
-wstrcmp.c: strcmp for mb
-wstrncmp.c: strncmp for mb
+ conv.c static functions and a public table for code conversion
+ mbutils.c public functions for the backend only.
+ stringinfo_mb.c public backend-only multibyte-aware stringinfo functions
+ wstrcmp.c strcmp for mb
+ wstrncmp.c strncmp for mb
See also in src/common/:
-encnames.c: public functions for encoding names
-wchar.c: mostly static functions and a public table for mb string and
- multibyte conversion
+ encnames.c public functions for encoding names
+ wchar.c mostly static functions and a public table for mb string and
+ multibyte conversion
Introduction
------------
diff --git a/src/backend/utils/misc/README b/src/backend/utils/misc/README
index 85d97d29b6..eaedfbc6df 100644
--- a/src/backend/utils/misc/README
+++ b/src/backend/utils/misc/README
@@ -23,29 +23,37 @@ modify the default SHOW display for a variable.
If a check_hook is provided, it points to a function of the signature
+
bool check_hook(datatype *newvalue, void **extra, GucSource source)
+
The "newvalue" argument is of type bool *, int *, double *, or char **
for bool, int/enum, real, or string variables respectively. The check
function should validate the proposed new value, and return true if it is
OK or false if not. The function can optionally do a few other things:
* When rejecting a bad proposed value, it may be useful to append some
-additional information to the generic "invalid value for parameter FOO"
-complaint that guc.c will emit. To do that, call
- void GUC_check_errdetail(const char *format, ...)
-where the format string and additional arguments follow the rules for
-errdetail() arguments. The resulting string will be emitted as the
-DETAIL line of guc.c's error report, so it should follow the message style
-guidelines for DETAIL messages. There is also
- void GUC_check_errhint(const char *format, ...)
-which can be used in the same way to append a HINT message.
-Occasionally it may even be appropriate to override guc.c's generic primary
-message or error code, which can be done with
- void GUC_check_errcode(int sqlerrcode)
- void GUC_check_errmsg(const char *format, ...)
-In general, check_hooks should avoid throwing errors directly if possible,
-though this may be impractical to avoid for some corner cases such as
-out-of-memory.
+ additional information to the generic "invalid value for parameter FOO"
+ complaint that guc.c will emit. To do that, call
+
+ void GUC_check_errdetail(const char *format, ...)
+
+ where the format string and additional arguments follow the rules for
+ errdetail() arguments. The resulting string will be emitted as the
+ DETAIL line of guc.c's error report, so it should follow the message style
+ guidelines for DETAIL messages. There is also
+
+ void GUC_check_errhint(const char *format, ...)
+
+ which can be used in the same way to append a HINT message.
+ Occasionally it may even be appropriate to override guc.c's generic primary
+ message or error code, which can be done with
+
+ void GUC_check_errcode(int sqlerrcode)
+ void GUC_check_errmsg(const char *format, ...)
+
+ In general, check_hooks should avoid throwing errors directly if possible,
+ though this may be impractical to avoid for some corner cases such as
+ out-of-memory.
* Since the newvalue is pass-by-reference, the function can modify it.
This might be used for example to canonicalize the spelling of a string
@@ -76,7 +84,9 @@ assignment will occur.
If an assign_hook is provided, it points to a function of the signature
+
void assign_hook(datatype newvalue, void *extra)
+
where the type of "newvalue" matches the kind of variable, and "extra"
is the derived-information pointer returned by the check_hook (always
NULL if there is no check_hook). This function is called immediately
@@ -110,7 +120,9 @@ needing to check GUC values outside a transaction.
If a show_hook is provided, it points to a function of the signature
+
const char *show_hook(void)
+
This hook allows variable-specific computation of the value displayed
by SHOW (and other SQL features for showing GUC variable values).
The return value can point to a static buffer, since show functions are
@@ -214,23 +226,23 @@ The merged entry will have level N-1 and prior = older prior, so easiest
to keep older entry and free newer. There are 12 possibilities since
we already handled level N state = SAVE:
-N-1 N
+ N-1 N
-SAVE SET discard top prior, set state SET
-SAVE LOCAL discard top prior, no change to stack entry
-SAVE SET+LOCAL discard top prior, copy masked, state S+L
+ SAVE SET discard top prior, set state SET
+ SAVE LOCAL discard top prior, no change to stack entry
+ SAVE SET+LOCAL discard top prior, copy masked, state S+L
-SET SET discard top prior, no change to stack entry
-SET LOCAL copy top prior to masked, state S+L
-SET SET+LOCAL discard top prior, copy masked, state S+L
+ SET SET discard top prior, no change to stack entry
+ SET LOCAL copy top prior to masked, state S+L
+ SET SET+LOCAL discard top prior, copy masked, state S+L
-LOCAL SET discard top prior, set state SET
-LOCAL LOCAL discard top prior, no change to stack entry
-LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ LOCAL SET discard top prior, set state SET
+ LOCAL LOCAL discard top prior, no change to stack entry
+ LOCAL SET+LOCAL discard top prior, copy masked, state S+L
-SET+LOCAL SET discard top prior and second masked, state SET
-SET+LOCAL LOCAL discard top prior, no change to stack entry
-SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
+ SET+LOCAL SET discard top prior and second masked, state SET
+ SET+LOCAL LOCAL discard top prior, no change to stack entry
+ SET+LOCAL SET+LOCAL discard top prior, copy masked, state S+L
RESET is executed like a SET, but using the reset_val as the desired new
diff --git a/src/backend/utils/mmgr/README b/src/backend/utils/mmgr/README
index 695088bb66..27a49a9a36 100644
--- a/src/backend/utils/mmgr/README
+++ b/src/backend/utils/mmgr/README
@@ -412,11 +412,11 @@ GetMemoryChunkMethodID() and finding the corresponding MemoryContextMethods
in the mcxt_methods[] array. For convenience, the MCXT_METHOD() macro is
provided, making the code as simple as:
-void
-pfree(void *pointer)
-{
- MCXT_METHOD(pointer, free_p)(pointer);
-}
+ void
+ pfree(void *pointer)
+ {
+ MCXT_METHOD(pointer, free_p)(pointer);
+ }
All of the current memory contexts make use of the MemoryChunk header type
which is defined in memutils_memorychunk.h. This suits all of the existing
diff --git a/src/backend/utils/resowner/README b/src/backend/utils/resowner/README
index cbf34e0b56..d0b4eb1d72 100644
--- a/src/backend/utils/resowner/README
+++ b/src/backend/utils/resowner/README
@@ -83,13 +83,13 @@ references, to name a few examples.
To add a new kind of resource, define a ResourceOwnerDesc to describe it.
For example:
-static const ResourceOwnerDesc myresource_desc = {
- .name = "My fancy resource",
- .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
- .release_priority = RELEASE_PRIO_FIRST,
- .ReleaseResource = ReleaseMyResource,
- .DebugPrint = PrintMyResource
-};
+ static const ResourceOwnerDesc myresource_desc = {
+ .name = "My fancy resource",
+ .release_phase = RESOURCE_RELEASE_AFTER_LOCKS,
+ .release_priority = RELEASE_PRIO_FIRST,
+ .ReleaseResource = ReleaseMyResource,
+ .DebugPrint = PrintMyResource
+ };
ResourceOwnerRemember() and ResourceOwnerForget() functions take a pointer
to that struct, along with a Datum to represent the resource. The meaning
@@ -139,28 +139,28 @@ within each phase.
For example, imagine that you have two ResourceOwners, parent and child,
as follows:
-Parent
- parent resource BEFORE_LOCKS priority 1
- parent resource BEFORE_LOCKS priority 2
- parent resource AFTER_LOCKS priority 10001
- parent resource AFTER_LOCKS priority 10002
- Child
- child resource BEFORE_LOCKS priority 1
- child resource BEFORE_LOCKS priority 2
- child resource AFTER_LOCKS priority 10001
- child resource AFTER_LOCKS priority 10002
+ Parent
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
+ Child
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
These resources would be released in the following order:
-child resource BEFORE_LOCKS priority 1
-child resource BEFORE_LOCKS priority 2
-parent resource BEFORE_LOCKS priority 1
-parent resource BEFORE_LOCKS priority 2
-(locks)
-child resource AFTER_LOCKS priority 10001
-child resource AFTER_LOCKS priority 10002
-parent resource AFTER_LOCKS priority 10001
-parent resource AFTER_LOCKS priority 10002
+ child resource BEFORE_LOCKS priority 1
+ child resource BEFORE_LOCKS priority 2
+ parent resource BEFORE_LOCKS priority 1
+ parent resource BEFORE_LOCKS priority 2
+ (locks)
+ child resource AFTER_LOCKS priority 10001
+ child resource AFTER_LOCKS priority 10002
+ parent resource AFTER_LOCKS priority 10001
+ parent resource AFTER_LOCKS priority 10002
To release all the resources, you need to call ResourceOwnerRelease() three
times, once for each phase. You may perform additional tasks between the
diff --git a/src/interfaces/ecpg/preproc/README.parser b/src/interfaces/ecpg/preproc/README.parser
index ddc3061d48..b647e248ab 100644
--- a/src/interfaces/ecpg/preproc/README.parser
+++ b/src/interfaces/ecpg/preproc/README.parser
@@ -14,29 +14,38 @@ ECPG modifies and extends the core grammar in a way that
actions for grammar rules.
In "ecpg.addons", every modified rule follows this pattern:
+
ECPG: dumpedtokens postfix
+
where "dumpedtokens" is simply tokens from core gram.y's
rules concatenated together. e.g. if gram.y has this:
- ruleA: tokenA tokenB tokenC {...}
+
+ ruleA: tokenA tokenB tokenC {...}
+
then "dumpedtokens" is "ruleAtokenAtokenBtokenC".
"postfix" above can be:
-a) "block" - the automatic rule created by parse.pl is completely
- overridden, the code block has to be written completely as
- it were in a plain bison grammar
-b) "rule" - the automatic rule is extended on, so new syntaxes
- are accepted for "ruleA". E.g.:
- ECPG: ruleAtokenAtokenBtokenC rule
- | tokenD tokenE { action_code; }
- ...
+
+1) "block" - the automatic rule created by parse.pl is completely
+ overridden, the code block has to be written completely as
+ it were in a plain bison grammar
+2) "rule" - the automatic rule is extended on, so new syntaxes
+ are accepted for "ruleA". E.g.:
+
+ ECPG: ruleAtokenAtokenBtokenC rule
+ | tokenD tokenE { action_code; }
+ ...
+
It will be substituted with:
- ruleA: <original syntax forms and actions up to and including
- "tokenA tokenB tokenC">
- | tokenD tokenE { action_code; }
- ...
-c) "addon" - the automatic action for the rule (SQL syntax constructed
- from the tokens concatenated together) is prepended with a new
- action code part. This code part is written as is's already inside
- the { ... }
+
+ ruleA: <original syntax forms and actions up to and including
+ "tokenA tokenB tokenC">
+ | tokenD tokenE { action_code; }
+ ...
+
+3) "addon" - the automatic action for the rule (SQL syntax constructed
+ from the tokens concatenated together) is prepended with a new
+ action code part. This code part is written as is's already inside
+ the { ... }
Multiple "addon" or "block" lines may appear together with the
new code block if the code block is common for those rules.
diff --git a/src/port/README b/src/port/README
index ed5c54a72f..6a8594aba8 100644
--- a/src/port/README
+++ b/src/port/README
@@ -14,15 +14,15 @@ libraries. This is done by removing -lpgport from the link line:
# Need to recompile any libpgport object files
LIBS := $(filter-out -lpgport, $(LIBS))
-and adding infrastructure to recompile the object files:
+ and adding infrastructure to recompile the object files:
OBJS= execute.o typename.o descriptor.o data.o error.o prepare.o memory.o \
connect.o misc.o path.o exec.o \
$(filter strlcat.o, $(LIBOBJS))
-The problem is that there is no testing of which object files need to be
-added, but missing functions usually show up when linking user
-applications.
+ The problem is that there is no testing of which object files need to be
+ added, but missing functions usually show up when linking user
+ applications.
2) For applications, we use -lpgport before -lpq, so the static files
from libpgport are linked first. This avoids having applications
diff --git a/src/test/isolation/README b/src/test/isolation/README
index 5818ca5003..471b0a5029 100644
--- a/src/test/isolation/README
+++ b/src/test/isolation/README
@@ -12,23 +12,33 @@ serializable isolation level; but tests for other sorts of concurrent
behaviors have been added as well.
You can run the tests against the current build tree by typing
+
make check
+
Alternatively, you can run against an existing installation by typing
+
make installcheck
+
(This will contact a server at the default port expected by libpq.
You can set PGPORT and so forth in your environment to control this.)
To run just specific test(s) against an installed server,
you can do something like
+
./pg_isolation_regress fk-contention fk-deadlock
+
(look into the specs/ subdirectory to see the available tests).
Certain tests require the server's max_prepared_transactions parameter to be
set to at least 3; therefore they are not run by default. To include them in
the test run, use
+
make check-prepared-txns
+
or
+
make installcheck-prepared-txns
+
after making sure the server configuration is correct (see TEMP_CONFIG
to adjust this in the "check" case).
@@ -64,7 +74,7 @@ that are to be run.
A test specification consists of four parts, in this order:
-setup { <SQL> }
+setup { < SQL > }
The given SQL block is executed once (per permutation) before running
the test. Create any test tables or other required objects here. This
@@ -74,13 +84,13 @@ setup { <SQL> }
and some statements such as VACUUM cannot be combined with others in such
a block.)
-teardown { <SQL> }
+teardown { < SQL > }
The teardown SQL block is executed once after the test is finished. Use
this to clean up in preparation for the next permutation, e.g dropping
any test tables created by setup. This part is optional.
-session <name>
+session < name >
There are normally several "session" parts in a spec file. Each
session is executed in its own connection. A session part consists
@@ -91,13 +101,13 @@ session <name>
Each step has the syntax
- step <name> { <SQL> }
+ step < name > { < SQL > }
- where <name> is a name identifying this step, and <SQL> is a SQL statement
+ where < name > is a name identifying this step, and < SQL > is a SQL statement
(or statements, separated by semicolons) that is executed in the step.
Step names must be unique across the whole spec file.
-permutation <step name> ...
+permutation < step name > ...
A permutation line specifies a list of steps that are run in that order.
Any number of permutation lines can appear. If no permutation lines are
@@ -116,10 +126,10 @@ whether you quote them or not. You must use quotes if you want to use
an isolation test keyword (such as "permutation") as a name.
A # character begins a comment, which extends to the end of the line.
-(This does not work inside <SQL> blocks, however. Use the usual SQL
+(This does not work inside < SQL > blocks, however. Use the usual SQL
comment conventions there.)
-There is no way to include a "}" character in an <SQL> block.
+There is no way to include a "}" character in an < SQL > block.
For each permutation of the session steps (whether these are manually
specified in the spec file, or automatically generated), the isolation
@@ -187,9 +197,9 @@ step has completed. (If the other step is used more than once in the
current permutation, this step cannot complete while any of those
instances is active.)
-A marker of the form "<other step name> notices <n>" (where <n> is a
+A marker of the form "< other step name > notices < n >" (where < n > is a
positive integer) indicates that this step may not be reported as
-completing until the other step's session has returned at least <n>
+completing until the other step's session has returned at least < n >
NOTICE messages, counting from when this step is launched. This is useful
for stabilizing cases where a step can return NOTICE messages before it
actually completes, and those messages must be synchronized with the
diff --git a/src/test/kerberos/README b/src/test/kerberos/README
index a048d442af..dc53747fd9 100644
--- a/src/test/kerberos/README
+++ b/src/test/kerberos/README
@@ -23,9 +23,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/dblink and contrib/postgres_fdw in addition to the core code.
Run
+
make check PG_TEST_EXTRA=kerberos
+
or
+
make installcheck PG_TEST_EXTRA=kerberos
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/locale/README b/src/test/locale/README
index e290e31480..eb6a7c6124 100644
--- a/src/test/locale/README
+++ b/src/test/locale/README
@@ -9,8 +9,11 @@ locale data. Then there are test-sort.pl and test-sort.py that test
collating.
To run a test for some locale run
+
make check-$locale
+
for example
+
make check-koi8-r
Currently, there are only tests for a few locales available. The script
diff --git a/src/test/modules/dummy_seclabel/README b/src/test/modules/dummy_seclabel/README
index a3fcbd7599..1a12fdaad2 100644
--- a/src/test/modules/dummy_seclabel/README
+++ b/src/test/modules/dummy_seclabel/README
@@ -18,13 +18,13 @@ Usage
Here's a simple example of usage:
-# postgresql.conf
-shared_preload_libraries = 'dummy_seclabel'
+ # postgresql.conf
+ shared_preload_libraries = 'dummy_seclabel'
-postgres=# CREATE TABLE t (a int, b text);
-CREATE TABLE
-postgres=# SECURITY LABEL ON TABLE t IS 'classified';
-SECURITY LABEL
+ postgres=# CREATE TABLE t (a int, b text);
+ CREATE TABLE
+ postgres=# SECURITY LABEL ON TABLE t IS 'classified';
+ SECURITY LABEL
The dummy_seclabel module provides only four hardcoded
labels: unclassified, classified,
diff --git a/src/test/modules/test_parser/README b/src/test/modules/test_parser/README
index 0a11ec85fb..735bfe9a00 100644
--- a/src/test/modules/test_parser/README
+++ b/src/test/modules/test_parser/README
@@ -5,12 +5,12 @@ a starting point for developing your own parser.
test_parser recognizes words separated by white space,
and returns just two token types:
-mydb=# SELECT * FROM ts_token_type('testparser');
- tokid | alias | description
--------+-------+---------------
- 3 | word | Word
- 12 | blank | Space symbols
-(2 rows)
+ mydb=# SELECT * FROM ts_token_type('testparser');
+ tokid | alias | description
+ -------+-------+---------------
+ 3 | word | Word
+ 12 | blank | Space symbols
+ (2 rows)
These token numbers have been chosen to be compatible with the default
parser's numbering. This allows us to use its headline()
@@ -24,38 +24,38 @@ parser testparser. It has no user-configurable parameters.
You can test the parser with, for example,
-mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
- tokid | token
--------+--------
- 3 | That's
- 12 |
- 3 | my
- 12 |
- 3 | first
- 12 |
- 3 | own
- 12 |
- 3 | parser
+ mydb=# SELECT * FROM ts_parse('testparser', 'That''s my first own parser');
+ tokid | token
+ -------+--------
+ 3 | That's
+ 12 |
+ 3 | my
+ 12 |
+ 3 | first
+ 12 |
+ 3 | own
+ 12 |
+ 3 | parser
Real-world use requires setting up a text search configuration
that uses the parser. For example,
-mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
-CREATE TEXT SEARCH CONFIGURATION
-
-mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
-mydb-# ADD MAPPING FOR word WITH english_stem;
-ALTER TEXT SEARCH CONFIGURATION
-
-mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
- to_tsvector
--------------------------------
- 'that':1 'first':3 'parser':5
-(1 row)
-
-mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
-mydb(# to_tsquery('testcfg', 'star'));
- ts_headline
------------------------------------------------------------------
- Supernovae <b>stars</b> are the brightest phenomena in galaxies
-(1 row)
+ mydb=# CREATE TEXT SEARCH CONFIGURATION testcfg ( PARSER = testparser );
+ CREATE TEXT SEARCH CONFIGURATION
+
+ mydb=# ALTER TEXT SEARCH CONFIGURATION testcfg
+ mydb-# ADD MAPPING FOR word WITH english_stem;
+ ALTER TEXT SEARCH CONFIGURATION
+
+ mydb=# SELECT to_tsvector('testcfg', 'That''s my first own parser');
+ to_tsvector
+ -------------------------------
+ 'that':1 'first':3 'parser':5
+ (1 row)
+
+ mydb=# SELECT ts_headline('testcfg', 'Supernovae stars are the brightest phenomena in galaxies',
+ mydb(# to_tsquery('testcfg', 'star'));
+ ts_headline
+ -----------------------------------------------------------------
+ Supernovae <b>stars</b> are the brightest phenomena in galaxies
+ (1 row)
diff --git a/src/test/modules/test_regex/README b/src/test/modules/test_regex/README
index 3ef152d4e1..7da5e67a9c 100644
--- a/src/test/modules/test_regex/README
+++ b/src/test/modules/test_regex/README
@@ -5,7 +5,7 @@ aren't currently exposed at the SQL level by PostgreSQL.
Currently, one function is provided:
-test_regex(pattern text, string text, flags text) returns setof text[]
+ test_regex(pattern text, string text, flags text) returns setof text[]
Reports an error if the pattern is an invalid regex. Otherwise,
the first row of output contains the number of subexpressions,
diff --git a/src/test/modules/test_rls_hooks/README b/src/test/modules/test_rls_hooks/README
index c22e0d3fb4..b561bb0192 100644
--- a/src/test/modules/test_rls_hooks/README
+++ b/src/test/modules/test_rls_hooks/README
@@ -3,14 +3,14 @@ define additional policies to be used.
Functions
=========
-test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_permissive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with OR.
-test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
- RETURNS List*
+ test_rls_hooks_restrictive(CmdType cmdtype, Relation relation)
+ RETURNS List*
Returns a list of policies which should be added to any existing
policies on the relation, combined with AND.
diff --git a/src/test/modules/test_shm_mq/README b/src/test/modules/test_shm_mq/README
index 641407bee0..1df3d38bde 100644
--- a/src/test/modules/test_shm_mq/README
+++ b/src/test/modules/test_shm_mq/README
@@ -14,9 +14,9 @@ Functions
=========
-test_shm_mq(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1)
- RETURNS void
+ test_shm_mq(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1)
+ RETURNS void
This function sends and receives messages synchronously. The user
backend sends the provided message to the first background worker using
@@ -31,10 +31,10 @@ the user backend verifies that the message finally received matches the
one originally sent and throws an error if not.
-test_shm_mq_pipelined(queue_size int8, message text,
- repeat_count int4 default 1, num_workers int4 default 1,
- verify bool default true)
- RETURNS void
+ test_shm_mq_pipelined(queue_size int8, message text,
+ repeat_count int4 default 1, num_workers int4 default 1,
+ verify bool default true)
+ RETURNS void
This function sends the same message multiple times, as specified by the
repeat count, to the first background worker using a queue of the given
diff --git a/src/test/recovery/README b/src/test/recovery/README
index 896df0ad05..d13156a140 100644
--- a/src/test/recovery/README
+++ b/src/test/recovery/README
@@ -14,9 +14,13 @@ contrib/pg_prewarm, contrib/pg_stat_statements and contrib/test_decoding
in addition to the core code.
Run
+
make check
+
or
+
make installcheck
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
diff --git a/src/test/ssl/README b/src/test/ssl/README
index 2101a466d2..c29f636c42 100644
--- a/src/test/ssl/README
+++ b/src/test/ssl/README
@@ -20,9 +20,13 @@ Also, to use "make installcheck", you must have built and installed
contrib/sslinfo in addition to the core code.
Run
+
make check PG_TEST_EXTRA=ssl
+
or
+
make installcheck PG_TEST_EXTRA=ssl
+
You can use "make installcheck" if you previously did "make install".
In that case, the code in the installation tree is tested. With
"make check", a temporary installation tree is built from the current
@@ -39,49 +43,49 @@ Certificates
The test suite needs a set of public/private key pairs and certificates to
run:
-root_ca
+* root_ca:
root CA, use to sign the server and client CA certificates.
-server_ca
+* server_ca:
CA used to sign server certificates.
-client_ca
+* client_ca:
CA used to sign client certificates.
-server-cn-only
-server-cn-and-alt-names
-server-single-alt-name
-server-multiple-alt-names
-server-no-names
+* server-cn-only,
+server-cn-and-alt-names,
+server-single-alt-name,
+server-multiple-alt-names,
+server-no-names:
server certificates, with small variations in the hostnames present
in the certificate. Signed by server_ca.
-server-password
+* server-password:
same as server-cn-only, but password-protected.
-client
+* client:
a client certificate, for user "ssltestuser". Signed by client_ca.
-client-revoked
+* client-revoked:
like "client", but marked as revoked in the client CA's CRL.
In addition, there are a few files that combine various certificates together
in the same file:
-both-cas-1
+* both-cas-1:
Contains root_ca.crt, client_ca.crt and server_ca.crt, in that order.
-both-cas-2
+* both-cas-2:
Contains root_ca.crt, server_ca.crt and client_ca.crt, in that order.
-root+server_ca
+* root+server_ca:
Contains root_crt and server_ca.crt. For use as client's "sslrootcert"
option.
-root+client_ca
+* root+client_ca:
Contains root_crt and client_ca.crt. For use as server's "ssl_ca_file".
-client+client_ca
+* client+client_ca:
Contains client.crt and client_ca.crt in that order. For use as client's
certificate chain.
diff --git a/src/timezone/tznames/README b/src/timezone/tznames/README
index 6d355e4616..3e05c80579 100644
--- a/src/timezone/tznames/README
+++ b/src/timezone/tznames/README
@@ -12,7 +12,7 @@ be a file here that serves your needs. If not, you can create your own.
In order to use one of these files, you need to set
- timezone_abbreviations = 'xyz'
+ timezone_abbreviations = 'xyz'
in any of the usual ways for setting a parameter, where xyz is the filename
that contains the desired time zone abbreviations.
diff --git a/src/tools/ci/README b/src/tools/ci/README
index 30ddd200c9..e82073ff8a 100644
--- a/src/tools/ci/README
+++ b/src/tools/ci/README
@@ -35,7 +35,7 @@ See also https://cirrus-ci.org/guide/quick-start/
Once enabled on a repository, future commits and pull-requests in that
repository will automatically trigger CI builds. These are visible from the
commit history / PRs, and can also be viewed in the cirrus-ci UI at
-https://cirrus-ci.com/github/<username>/<reponame>/
+https://cirrus-ci.com/github/< username >/< reponame >/
Hint: all build log files are uploaded to cirrus-ci and can be downloaded
from the "Artifacts" section from the cirrus-ci UI after clicking into a
@@ -74,7 +74,7 @@ When running a lot of tests in a repository, cirrus-ci's free credits do not
suffice. In those cases a repository can be configured to use other
infrastructure for running tests. To do so, the REPO_CI_CONFIG_GIT_URL
variable can be configured for the repository in the cirrus-ci web interface,
-at https://cirrus-ci.com/github/<user or organization>. The file referenced
+at https://cirrus-ci.com/github/< user or organization >. The file referenced
(see https://cirrus-ci.org/guide/programming-tasks/#fs) by the variable can
overwrite the default execution method for different operating systems,
defined in .cirrus.yml, by redefining the relevant yaml anchors.
diff --git a/src/tools/pgindent/README b/src/tools/pgindent/README
index b6cd4c6f6b..ac533481c0 100644
--- a/src/tools/pgindent/README
+++ b/src/tools/pgindent/README
@@ -10,7 +10,7 @@ http://adpgtech.blogspot.com/2015/05/running-pgindent-on-non-core-code-or.html
PREREQUISITES:
-
+--------------
1) Install pg_bsd_indent in your PATH. Its source code is in the
sibling directory src/tools/pg_bsd_indent; see the directions
in that directory's README file.
@@ -22,18 +22,21 @@ PREREQUISITES:
To install, follow the usual install process for a Perl module
("man perlmodinstall" explains it). Or, if you have cpan installed,
this should work:
- cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+
+ cpan SHANCOCK/Perl-Tidy-20230309.tar.gz
+
Or if you have cpanm installed, you can just use:
- cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
+ cpanm https://cpan.metacpan.org/authors/id/S/SH/SHANCOCK/Perl-Tidy-20230309.tar.gz
-DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
+DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
+--------------------------------------------
1) Change directory to the top of the source tree.
2) Run pgindent on the C files:
- src/tools/pgindent/pgindent .
+ src/tools/pgindent/pgindent .
If any files generate errors, restore their original versions with
"git checkout", and see below for cleanup ideas.
@@ -52,9 +55,9 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
6) Do a full test build:
- make -s clean
- make -s all # look for unexpected warnings, and errors of course
- make check-world
+ make -s clean
+ make -s all # look for unexpected warnings, and errors of course
+ make check-world
Your configure switches should include at least --enable-tap-tests
or else much of the Perl code won't get exercised.
@@ -64,10 +67,10 @@ DOING THE INDENT RUN BEFORE A NORMAL COMMIT:
AT LEAST ONCE PER RELEASE CYCLE:
-
+--------------------------------
1) Download the latest typedef file from the buildfarm:
- wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
+ wget -O src/tools/pgindent/typedefs.list https://buildfarm.postgresql.org/cgi-bin/typedefs.pl
This step resolves any differences between the incrementally updated
version of the file and a clean, autogenerated one.
@@ -78,17 +81,17 @@ AT LEAST ONCE PER RELEASE CYCLE:
3) Indent the Perl code using perltidy:
- src/tools/pgindent/pgperltidy .
+ src/tools/pgindent/pgperltidy .
If you want to use some perltidy version that's not in your PATH,
first set the PERLTIDY environment variable to point to it.
4) Reformat the bootstrap catalog data files:
- ./configure # "make" will not work in an unconfigured tree
- cd src/include/catalog
- make reformat-dat-files
- cd ../../..
+ ./configure # "make" will not work in an unconfigured tree
+ cd src/include/catalog
+ make reformat-dat-files
+ cd ../../..
5) When you're done, "git commit" everything including the typedefs.list file
you used.
--
2.39.3 (Apple Git-146)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-10-01 14:53 Jelte Fennema-Nio <[email protected]>
parent: Daniel Gustafsson <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Jelte Fennema-Nio @ 2024-10-01 14:53 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; PostgreSQL Developers <[email protected]>
On Tue, 1 Oct 2024 at 15:52, Daniel Gustafsson <[email protected]> wrote:
> > So we need to think about a way to make this more robust for future people editing. Maybe something in .gitattributes or some editor settings. Otherwise, it will be all over the places after a while.
>
> Maybe we can add some form of pandoc target for rendering as as way to test
> locally before pushing?
I think a gitattributes rule to disallow hard-tabs word work fine,
especially when combined with this patch of mine which keeps the
.editorconfig file in sync with the .gitattributes file:
https://commitfest.postgresql.org/49/4829/
> > Apart from this, I don't changing the placeholders like <foo> to < foo >. In some cases, this really decreases readability. Maybe we should look for different approaches there.
>
> Agreed. I took a stab at some of them in the attached. The usage in
> src/test/isolation/README is seemingly the hardest to replace and I'm not sure
> how we should proceed there.
One way to improve the isolation/README situation is by:
1. indenting the standalone lines by four spaces to make it a code block
2. for the inline cases, replace <foo> with `<foo>` or `foo`
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-10-01 20:15 Daniel Gustafsson <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 33+ messages in thread
From: Daniel Gustafsson @ 2024-10-01 20:15 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; PostgreSQL Developers <[email protected]>
> On 1 Oct 2024, at 16:53, Jelte Fennema-Nio <[email protected]> wrote:
> On Tue, 1 Oct 2024 at 15:52, Daniel Gustafsson <[email protected]> wrote:
>>> Apart from this, I don't changing the placeholders like <foo> to < foo >. In some cases, this really decreases readability. Maybe we should look for different approaches there.
>>
>> Agreed. I took a stab at some of them in the attached. The usage in
>> src/test/isolation/README is seemingly the hardest to replace and I'm not sure
>> how we should proceed there.
>
> One way to improve the isolation/README situation is by:
> 1. indenting the standalone lines by four spaces to make it a code block
> 2. for the inline cases, replace <foo> with `<foo>` or `foo`
If we go for following Markdown syntax then for sure, if not it will seem a bit
off I think.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-10-02 04:17 Tristan Partin <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Tristan Partin @ 2024-10-02 04:17 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; Peter Eisentraut <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; PostgreSQL Developers <[email protected]>
On Sat Jun 29, 2024 at 5:24 AM EDT, Jelte Fennema-Nio wrote:
> On Fri, 28 Jun 2024 at 20:40, Peter Eisentraut <[email protected]> wrote:
>> I have my "less" set up so that "less somefile.md" automatically renders
>> the markdown. That's been pretty useful. But if that now keeps making
>> a mess out of PostgreSQL's README files, then I'm going to have to keep
>> fixing things, and I might get really mad. That's the worst that could
>> happen. ;-)
>
> Do you have reason to think that this is going to be a bigger issue
> for Postgres READMEs than for any other markdown files you encounter?
> Because this sounds like a generic problem you'd run into with your
> "less" set up, which so far apparently has been small enough that it's
> worth the benefit of automatically rendering markdown files.
>
>> So I don't agree with "aspirational markdown". If we're going to do it,
>> then I expect that the files are marked up correctly at all times.
>
> I think for at least ~90% of our README files this shouldn't be a
> problem. If you have specific ones in mind that contain difficult
> markup/diagrams, then maybe we shouldn't convert those.
>
>> Conversely, what's the best that could happen?
>
> That your "less" would automatically render Postgres READMEs nicely.
> Which you say has been pretty useful ;-) And maybe even show syntax
> highlighting for codeblocks.
>
> P.S. Now I'm wondering what your "less" is.
I'm also interested in your config, Peter. I've been using glow[0] to
render markdown in my terminal, but I don't have that configured with
less(1)[1].
[0]: https://github.com/charmbracelet/glow
[1]: https://git.sr.ht/~tristan957/dotfiles/tree/5a267b7e0de260f814a1830f98c55d1accb9d72e/item/less/.conf...
--
Tristan Partin
Neon (https://neon.tech)
^ permalink raw reply [nested|flat] 33+ messages in thread
* Re: Converting README documentation to Markdown
@ 2024-12-02 09:02 Peter Eisentraut <[email protected]>
parent: Daniel Gustafsson <[email protected]>
0 siblings, 0 replies; 33+ messages in thread
From: Peter Eisentraut @ 2024-12-02 09:02 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; Jelte Fennema-Nio <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>
On 01.10.24 22:15, Daniel Gustafsson wrote:
>> On 1 Oct 2024, at 16:53, Jelte Fennema-Nio <[email protected]> wrote:
>> On Tue, 1 Oct 2024 at 15:52, Daniel Gustafsson <[email protected]> wrote:
>
>>>> Apart from this, I don't changing the placeholders like <foo> to < foo >. In some cases, this really decreases readability. Maybe we should look for different approaches there.
>>>
>>> Agreed. I took a stab at some of them in the attached. The usage in
>>> src/test/isolation/README is seemingly the hardest to replace and I'm not sure
>>> how we should proceed there.
>>
>> One way to improve the isolation/README situation is by:
>> 1. indenting the standalone lines by four spaces to make it a code block
>> 2. for the inline cases, replace <foo> with `<foo>` or `foo`
>
> If we go for following Markdown syntax then for sure, if not it will seem a bit
> off I think.
I took another look through this discussion. I think the v4 patches
from 2024-10-01 are a good improvement. I suggest you commit them and
then we can be done here.
^ permalink raw reply [nested|flat] 33+ messages in thread
end of thread, other threads:[~2024-12-02 09:02 UTC | newest]
Thread overview: 33+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-07-05 14:24 [PATCH 06/17] Allow user to use password instead of encryption key. Antonin Houska <[email protected]>
2023-07-20 17:19 [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v6 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v8 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v9 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-07-20 17:19 [PATCH v10 3/4] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-09-04 22:35 [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-09-04 22:35 [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2023-09-04 22:35 [PATCH v7 4/5] Convert pg_restore's ready_list to a priority queue. Nathan Bossart <[email protected]>
2024-05-13 07:20 Re: Converting README documentation to Markdown Peter Eisentraut <[email protected]>
2024-05-15 12:26 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-06-28 07:38 ` Re: Converting README documentation to Markdown Peter Eisentraut <[email protected]>
2024-06-28 09:56 ` Re: Converting README documentation to Markdown Jelte Fennema-Nio <[email protected]>
2024-06-28 18:40 ` Re: Converting README documentation to Markdown Peter Eisentraut <[email protected]>
2024-06-29 09:24 ` Re: Converting README documentation to Markdown Jelte Fennema-Nio <[email protected]>
2024-10-02 04:17 ` Re: Converting README documentation to Markdown Tristan Partin <[email protected]>
2024-07-01 09:42 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-07-01 10:22 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-09-10 12:50 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-09-10 15:37 ` Re: Converting README documentation to Markdown Tom Lane <[email protected]>
2024-09-10 18:25 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-09-23 11:58 ` Re: Converting README documentation to Markdown Peter Eisentraut <[email protected]>
2024-10-01 13:51 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-10-01 14:53 ` Re: Converting README documentation to Markdown Jelte Fennema-Nio <[email protected]>
2024-10-01 20:15 ` Re: Converting README documentation to Markdown Daniel Gustafsson <[email protected]>
2024-12-02 09:02 ` Re: Converting README documentation to Markdown Peter Eisentraut <[email protected]>
2024-06-28 12:37 ` Re: Converting README documentation to Markdown Tatsuo Ishii <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox