From: Alexey Kondratov <a.kondratov@postgrespro.ru>
To: Dmitry Dolgov <9erthalion6@gmail.com>
Cc: Michael Paquier <michael@paquier.xyz>
Cc: Alvaro Herrera <alvherre@2ndquadrant.com>
Cc: x4mmm@yandex-team.ru, PostgreSQL-development <pgsql-hackers@postgresql.org>
Cc: vladimirlesk@yandex-team.ru
Cc: dsarafan@yandex-team.ru
Subject: Re: [Patch] pg_rewind: options to use restore_command from recovery.conf or command line
Date: Mon, 24 Dec 2018 19:31:38 +0300
Message-ID: <13ec67dc-2c6b-777e-902a-1141fdb8302e@postgrespro.ru> (raw)
In-Reply-To: <CA+q6zcUw9KaV-L3khD9axOHU8x+sxota_pQP8-LqqFiOtgQmTA@mail.gmail.com>
References: <50b94dcb-c744-ff2c-d6a4-3c371f8a7f9a@postgrespro.ru>
<20181022171907.ri2tkqtt7n6jm4dh@alvherre.pgsql>
<20181029070600.GJ14242@paquier.xyz>
<527530a0-3460-7c32-d223-3014ddaeee8e@postgrespro.ru>
<20181030030123.GA18366@paquier.xyz>
<3e432374-d21b-9ab7-06e0-a0785e5e3fd0@postgrespro.ru>
<CA+q6zcUw9KaV-L3khD9axOHU8x+sxota_pQP8-LqqFiOtgQmTA@mail.gmail.com>
Hi Dmitry,
On 30.11.2018 19:04, Dmitry Dolgov wrote:
> Just to confirm, patch still can be applied without conflicts, and pass all the
> tests. Also I like the original motivation for the feature, sounds pretty
> useful. For now I'm moving it to the next CF.
Thanks, although I have slightly updated patch to handle recent merge of
the recovery.conf into GUCs and postgresq.conf [1], new patch is attached.
>>
>>> - Reusing the GUC parser is something I would avoid as well. Not worth
>>> the complexity.
>> Yes, I don't like it either. I will try to make guc-file.l frontend safe.
> Any success with that?
I looked into it and found that currently guc-file.c is built as part of
guc.c, so it seems to be even more complicated to unbound guc-file.c
from backend. Thus, I have some plan of how to proceed with patch:
1) Add guc-file.h and build guc-file.c separately from guc.c
2) Put guc-file.l / guc-file.h into common/*
3) Isolate all backend specific calls in guc-file.l with #ifdef FRONTEND
Though I am not sure that this work is worth doing against extra
redundancy added by simply adding frontend-safe copy of guc-file.l
lexer. If someone has any thoughts I would be glad to receive comments.
[1]
https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=2dedf4d9a899b36d1a8ed29be5efbd1b3...
Regards
--
Alexey Kondratov
Postgres Professional https://www.postgrespro.com
Russian Postgres Company
Attachments:0001-pg_rewind-options-to-use-restore_command-v1.2.patch (text/x-patch, 40.0K) download
From 521f62872d4e95cd02ddb535b8320256ff5e90cc Mon Sep 17 00:00:00 2001
From: Alexey Kondratov <alex.lumir@gmail.com>
Date: Fri, 21 Dec 2018 14:00:30 +0300
Subject: [PATCH] pg_rewind: options to use restore_command from
postgresql.conf or command line.
---
src/bin/pg_rewind/Makefile | 5 +-
src/bin/pg_rewind/RewindTest.pm | 46 +-
src/bin/pg_rewind/guc-file-fe.h | 40 ++
src/bin/pg_rewind/guc-file-fe.l | 776 ++++++++++++++++++++++++++
src/bin/pg_rewind/parsexlog.c | 182 +++++-
src/bin/pg_rewind/pg_rewind.c | 91 ++-
src/bin/pg_rewind/pg_rewind.h | 10 +-
src/bin/pg_rewind/t/001_basic.pl | 3 +-
src/bin/pg_rewind/t/002_databases.pl | 3 +-
src/bin/pg_rewind/t/003_extrafiles.pl | 3 +-
src/tools/msvc/Mkvcbuild.pm | 1 +
11 files changed, 1141 insertions(+), 19 deletions(-)
create mode 100644 src/bin/pg_rewind/guc-file-fe.h
create mode 100644 src/bin/pg_rewind/guc-file-fe.l
diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefileindex 2bcfcc61af..a0f5f97544 100644--- a/src/bin/pg_rewind/Makefile+++ b/src/bin/pg_rewind/Makefile@@ -15,11 +15,12 @@ subdir = src/bin/pg_rewind
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS)+override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS)
LDFLAGS_INTERNAL += $(libpq_pgport)
OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \
fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o \
+ guc-file-fe.o \
$(WIN32RES)
EXTRA_CLEAN = xlogreader.c
@@ -32,6 +33,8 @@ pg_rewind: $(OBJS) | submake-libpq submake-libpgport
xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
rm -f $@ && $(LN_S) $< .
+distprep: guc-file-fe.c+
install: all installdirs
$(INSTALL_PROGRAM) pg_rewind$(X) '$(DESTDIR)$(bindir)/pg_rewind$(X)'
diff --git a/src/bin/pg_rewind/RewindTest.pm b/src/bin/pg_rewind/RewindTest.pmindex 3d07da5d94..b43c18a8c3 100644--- a/src/bin/pg_rewind/RewindTest.pm+++ b/src/bin/pg_rewind/RewindTest.pm@@ -39,7 +39,9 @@ use Carp;
use Config;
use Exporter 'import';
use File::Copy;
-use File::Path qw(rmtree);+use File::Glob ':bsd_glob';+use File::Path qw(remove_tree make_path);+use File::Spec::Functions 'catpath';
use IPC::Run qw(run);
use PostgresNode;
use TestLib;
@@ -250,6 +252,48 @@ sub run_pg_rewind
],
'pg_rewind remote');
}
+ elsif ($test_mode eq "archive")+ {++ # Do rewind using a local pgdata as source and+ # specified directory with target WALs archive.+ my $wals_archive_dir = catpath(${TestLib::tmp_check}, 'master_wals_archive');+ my $test_master_datadir = $node_master->data_dir;+ my @wal_files = bsd_glob catpath($test_master_datadir, 'pg_wal', '0000000*');+ my $restore_command;++ remove_tree($wals_archive_dir);+ make_path($wals_archive_dir) or die;++ # Move all old master WAL files to the archive.+ # Old master should be stopped at this point.+ foreach my $wal_file (@wal_files)+ {+ move($wal_file, "$wals_archive_dir/") or die;+ }++ if ($windows_os)+ {+ $restore_command = "copy $wals_archive_dir\\\%f \%p";+ }+ else+ {+ $restore_command = "cp $wals_archive_dir/\%f \%p";+ }++ # Stop the new master and be ready to perform the rewind.+ $node_standby->stop;+ command_ok(+ [+ 'pg_rewind',+ "--debug",+ "--source-pgdata=$standby_pgdata",+ "--target-pgdata=$master_pgdata",+ "--no-sync",+ "-R", $restore_command+ ],+ 'pg_rewind archive');+ }
else
{
diff --git a/src/bin/pg_rewind/guc-file-fe.h b/src/bin/pg_rewind/guc-file-fe.hnew file mode 100644index 0000000000..3a078c5cdd--- /dev/null+++ b/src/bin/pg_rewind/guc-file-fe.h@@ -0,0 +1,40 @@+#ifndef PG_REWIND_GUC_FILE_FE_H+#define PG_REWIND_GUC_FILE_FE_H++#include "c.h"++#define RECOVERY_COMMAND_FILE "postgresql.conf"++/*+ * Parsing the configuration file(s) will return a list of name-value pairs+ * with source location info. We also abuse this data structure to carry+ * error reports about the config files. An entry reporting an error will+ * have errmsg != NULL, and might have NULLs for name, value, and/or filename.+ *+ * If "ignore" is true, don't attempt to apply the item (it might be an error+ * report, or an item we determined to be duplicate). "applied" is set true+ * if we successfully applied, or could have applied, the setting.+ */+typedef struct ConfigVariable+{+ char *name;+ char *value;+ char *errmsg;+ char *filename;+ int sourceline;+ bool ignore;+ bool applied;+ struct ConfigVariable *next;+} ConfigVariable;++extern bool ParseConfigFile(const char *config_file, bool strict,+ const char *calling_file, int calling_lineno,+ int depth, int elevel,+ ConfigVariable **head_p, ConfigVariable **tail_p);++extern bool ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,+ ConfigVariable **head_p, ConfigVariable **tail_p);++extern void FreeConfigVariables(ConfigVariable *list);++#endif /* PG_REWIND_GUC_FILE_FE_H */diff --git a/src/bin/pg_rewind/guc-file-fe.l b/src/bin/pg_rewind/guc-file-fe.lnew file mode 100644index 0000000000..b7ef47111a--- /dev/null+++ b/src/bin/pg_rewind/guc-file-fe.l@@ -0,0 +1,776 @@+/* -*-pgsql-c-*- */+/*+ * Configuration files parser for usage in frontend.+ *+ * Copyright (c) 2000-2018, PostgreSQL Global Development Group+ *+ * src/bin/pg_rewind/guc-file-fe.l+ * Modified version of src/backend/utils/misc/guc-file.l+ */++%{++#include "postgres_fe.h"++#include <ctype.h>+#include <dirent.h>+#include <unistd.h>+#include <setjmp.h>+#include <sys/types.h>+#include <sys/stat.h>++#include "guc-file-fe.h"+#include "logging.h"++#include "common/fe_memutils.h"++/*+ * flex emits a yy_fatal_error() function that it calls in response to+ * critical errors like malloc failure, file I/O errors, and detection of+ * internal inconsistency. That function prints a message and calls exit().+ * Mutate it to instead call our handler, which jumps out of the parser.+ */+#undef fprintf+#define fprintf(file, fmt, msg) GUC_flex_fatal(msg)++enum+{+ GUC_ID = 1,+ GUC_STRING = 2,+ GUC_INTEGER = 3,+ GUC_REAL = 4,+ GUC_EQUALS = 5,+ GUC_UNQUOTED_STRING = 6,+ GUC_QUALIFIED_ID = 7,+ GUC_EOL = 99,+ GUC_ERROR = 100+};++static unsigned int ConfigFileLineno;+static const char *GUC_flex_fatal_errmsg;+static sigjmp_buf *GUC_flex_fatal_jmp;++static void FreeConfigVariable(ConfigVariable *item);++static void record_config_file_error(const char *errmsg,+ const char *config_file,+ int lineno,+ ConfigVariable **head_p,+ ConfigVariable **tail_p);++static int GUC_flex_fatal(const char *msg);+static char *GUC_scanstr(const char *s);+static bool ParseConfigDirectory(const char *includedir,+ const char *calling_file, int calling_lineno,+ int depth, int elevel,+ ConfigVariable **head_p,+ ConfigVariable **tail_p);++/* LCOV_EXCL_START */++%}++%option 8bit+%option never-interactive+%option nodefault+%option noinput+%option nounput+%option noyywrap+%option warn+%option prefix="GUC_yy"+++SIGN ("-"|"+")+DIGIT [0-9]+HEXDIGIT [0-9a-fA-F]++UNIT_LETTER [a-zA-Z]++INTEGER {SIGN}?({DIGIT}+|0x{HEXDIGIT}+){UNIT_LETTER}*++EXPONENT [Ee]{SIGN}?{DIGIT}++REAL {SIGN}?{DIGIT}*"."{DIGIT}*{EXPONENT}?++LETTER [A-Za-z_\200-\377]+LETTER_OR_DIGIT [A-Za-z_0-9\200-\377]++ID {LETTER}{LETTER_OR_DIGIT}*+QUALIFIED_ID {ID}"."{ID}++UNQUOTED_STRING {LETTER}({LETTER_OR_DIGIT}|[-._:/])*+STRING \'([^'\\\n]|\\.|\'\')*\'++%%++\n ConfigFileLineno++; return GUC_EOL;+[ \t\r]+ /* eat whitespace */+#.* /* eat comment (.* matches anything until newline) */++{ID} return GUC_ID;+{QUALIFIED_ID} return GUC_QUALIFIED_ID;+{STRING} return GUC_STRING;+{UNQUOTED_STRING} return GUC_UNQUOTED_STRING;+{INTEGER} return GUC_INTEGER;+{REAL} return GUC_REAL;+= return GUC_EQUALS;++. return GUC_ERROR;++%%++/* LCOV_EXCL_STOP */++/*+ * the bare comparison function for GUC names+ */+static int+guc_name_compare(const char *namea, const char *nameb)+{+ /*+ * The temptation to use strcasecmp() here must be resisted, because the+ * array ordering has to remain stable across setlocale() calls. So, build+ * our own with a simple ASCII-only downcasing.+ */+ while (*namea && *nameb)+ {+ char cha = *namea++;+ char chb = *nameb++;++ if (cha >= 'A' && cha <= 'Z')+ cha += 'a' - 'A';+ if (chb >= 'A' && chb <= 'Z')+ chb += 'a' - 'A';+ if (cha != chb)+ return cha - chb;+ }+ if (*namea)+ return 1; /* a is longer */+ if (*nameb)+ return -1; /* b is longer */+ return 0;+}++/*+ * Given a configuration file or directory location that may be a relative+ * path, return an absolute one. We consider the location to be relative to+ * the directory holding the calling file.+ */+static char *+AbsoluteConfigLocation(const char *location, const char *calling_file)+{+ char abs_path[MAXPGPATH];++ if (is_absolute_path(location))+ return pstrdup(location);+ else+ {+ strlcpy(abs_path, calling_file, sizeof(abs_path));+ get_parent_directory(abs_path);+ join_path_components(abs_path, abs_path, location);+ canonicalize_path(abs_path);++ return pstrdup(abs_path);+ }+}++/*+ * Read and parse a single configuration file. This function recurses+ * to handle "include" directives.+ *+ * If "strict" is true, treat failure to open the config file as an error,+ * otherwise just skip the file.+ *+ * calling_file/calling_lineno identify the source of the request.+ * Pass NULL/0 if not recursing from an inclusion request.+ *+ * See ParseConfigFp for further details. This one merely adds opening the+ * config file rather than working from a caller-supplied file descriptor,+ * and absolute-ifying the path name if necessary.+ */+bool+ParseConfigFile(const char *config_file, bool strict,+ const char *calling_file, int calling_lineno,+ int depth, int elevel,+ ConfigVariable **head_p,+ ConfigVariable **tail_p)+{+ char *abs_path;+ bool OK = true;+ FILE *fp;++ /*+ * Reject too-deep include nesting depth. This is just a safety check to+ * avoid dumping core due to stack overflow if an include file loops back+ * to itself. The maximum nesting depth is pretty arbitrary.+ */+ if (depth > 10)+ {+ pg_log(PG_WARNING, + "could not open configuration file \"%s\": maximum nesting depth exceeded\n",+ config_file);+ record_config_file_error("nesting depth exceeded",+ calling_file, calling_lineno,+ head_p, tail_p);+ return false;+ }++ abs_path = AbsoluteConfigLocation(config_file, calling_file);+ fp = fopen(abs_path, "r");+ if (!fp)+ {+ if (strict)+ {+ pg_log(PG_WARNING, + "could not open configuration file \"%s\"\n",+ abs_path);+ record_config_file_error(psprintf("could not open file \"%s\"",+ abs_path),+ calling_file, calling_lineno,+ head_p, tail_p);+ OK = false;+ }+ else+ {+ pg_log(PG_WARNING, + "skipping missing configuration file \"%s\"\n",+ abs_path);+ }+ goto cleanup;+ }++ OK = ParseConfigFp(fp, abs_path, depth, elevel, head_p, tail_p);++cleanup:+ if (fp)+ fclose(fp);+ pfree(abs_path);++ return OK;+}++/*+ * Capture an error message in the ConfigVariable list returned by+ * config file parsing.+ */+static void+record_config_file_error(const char *errmsg,+ const char *config_file,+ int lineno,+ ConfigVariable **head_p,+ ConfigVariable **tail_p)+{+ ConfigVariable *item;++ item = palloc(sizeof *item);+ item->name = NULL;+ item->value = NULL;+ item->errmsg = pstrdup(errmsg);+ item->filename = config_file ? pstrdup(config_file) : NULL;+ item->sourceline = lineno;+ item->ignore = true;+ item->applied = false;+ item->next = NULL;+ if (*head_p == NULL)+ *head_p = item;+ else+ (*tail_p)->next = item;+ *tail_p = item;+}++/*+ * Flex fatal errors bring us here. Stash the error message and jump back to+ * ParseConfigFp(). Assume all msg arguments point to string constants; this+ * holds for flex 2.5.31 (earliest we support) and flex 2.5.35 (latest as of+ * this writing). Otherwise, we would need to copy the message.+ *+ * We return "int" since this takes the place of calls to fprintf().+*/+static int+GUC_flex_fatal(const char *msg)+{+ GUC_flex_fatal_errmsg = msg;+ siglongjmp(*GUC_flex_fatal_jmp, 1);+ return 0; /* keep compiler quiet */+}++/*+ * Read and parse a single configuration file. This function recurses+ * to handle "include" directives.+ *+ * Input parameters:+ * fp: file pointer from AllocateFile for the configuration file to parse+ * config_file: absolute or relative path name of the configuration file+ * depth: recursion depth (should be 0 in the outermost call)+ * elevel: error logging level to use+ * Input/Output parameters:+ * head_p, tail_p: head and tail of linked list of name/value pairs+ *+ * *head_p and *tail_p must be initialized, either to NULL or valid pointers+ * to a ConfigVariable list, before calling the outer recursion level. Any+ * name-value pairs read from the input file(s) will be appended to the list.+ * Error reports will also be appended to the list, if elevel < ERROR.+ *+ * Returns TRUE if successful, FALSE if an error occurred.+ *+ * Note: this function is used to parse not only postgresql.conf, but+ * various other configuration files that use the same "name = value"+ * syntax. Hence, do not do anything here or in the subsidiary routines+ * ParseConfigFile/ParseConfigDirectory that assumes we are processing+ * GUCs specifically.+ */+bool+ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,+ ConfigVariable **head_p, ConfigVariable **tail_p)+{+ volatile bool OK = true;+ unsigned int save_ConfigFileLineno = ConfigFileLineno;+ sigjmp_buf *save_GUC_flex_fatal_jmp = GUC_flex_fatal_jmp;+ sigjmp_buf flex_fatal_jmp;+ volatile YY_BUFFER_STATE lex_buffer = NULL;+ int errorcount;+ int token;++ if (sigsetjmp(flex_fatal_jmp, 1) == 0)+ GUC_flex_fatal_jmp = &flex_fatal_jmp;+ else+ {+ /*+ * Regain control after a fatal, internal flex error. It may have+ * corrupted parser state. Consequently, abandon the file, but trust+ * that the state remains sane enough for yy_delete_buffer().+ */+ pg_log(PG_WARNING, "%s at file \"%s\" line %u\n", GUC_flex_fatal_errmsg,+ config_file, ConfigFileLineno);+ record_config_file_error(GUC_flex_fatal_errmsg,+ config_file, ConfigFileLineno,+ head_p, tail_p);+ OK = false;+ goto cleanup;+ }++ /*+ * Parse+ */+ ConfigFileLineno = 1;+ errorcount = 0;++ lex_buffer = yy_create_buffer(fp, YY_BUF_SIZE);+ yy_switch_to_buffer(lex_buffer);++ /* This loop iterates once per logical line */+ while ((token = yylex()))+ {+ char *opt_name = NULL;+ char *opt_value = NULL;+ ConfigVariable *item;++ if (token == GUC_EOL) /* empty or comment line */+ continue;++ /* first token on line is option name */+ if (token != GUC_ID && token != GUC_QUALIFIED_ID)+ goto parse_error;+ opt_name = pstrdup(yytext);++ /* next we have an optional equal sign; discard if present */+ token = yylex();+ if (token == GUC_EQUALS)+ token = yylex();++ /* now we must have the option value */+ if (token != GUC_ID &&+ token != GUC_STRING &&+ token != GUC_INTEGER &&+ token != GUC_REAL &&+ token != GUC_UNQUOTED_STRING)+ goto parse_error;+ if (token == GUC_STRING) /* strip quotes and escapes */+ opt_value = GUC_scanstr(yytext);+ else+ opt_value = pstrdup(yytext);++ /* now we'd like an end of line, or possibly EOF */+ token = yylex();+ if (token != GUC_EOL)+ {+ if (token != 0)+ goto parse_error;+ /* treat EOF like \n for line numbering purposes, cf bug 4752 */+ ConfigFileLineno++;+ }++ /* OK, process the option name and value */+ if (guc_name_compare(opt_name, "include_dir") == 0)+ {+ /*+ * An include_dir directive isn't a variable and should be+ * processed immediately.+ */+ if (!ParseConfigDirectory(opt_value,+ config_file, ConfigFileLineno - 1,+ depth + 1, elevel,+ head_p, tail_p))+ OK = false;+ yy_switch_to_buffer(lex_buffer);+ pfree(opt_name);+ pfree(opt_value);+ }+ else if (guc_name_compare(opt_name, "include_if_exists") == 0)+ {+ /*+ * An include_if_exists directive isn't a variable and should be+ * processed immediately.+ */+ if (!ParseConfigFile(opt_value, false,+ config_file, ConfigFileLineno - 1,+ depth + 1, elevel,+ head_p, tail_p))+ OK = false;+ yy_switch_to_buffer(lex_buffer);+ pfree(opt_name);+ pfree(opt_value);+ }+ else if (guc_name_compare(opt_name, "include") == 0)+ {+ /*+ * An include directive isn't a variable and should be processed+ * immediately.+ */+ if (!ParseConfigFile(opt_value, true,+ config_file, ConfigFileLineno - 1,+ depth + 1, elevel,+ head_p, tail_p))+ OK = false;+ yy_switch_to_buffer(lex_buffer);+ pfree(opt_name);+ pfree(opt_value);+ }+ else+ {+ /* ordinary variable, append to list */+ item = palloc(sizeof *item);+ item->name = opt_name;+ item->value = opt_value;+ item->errmsg = NULL;+ item->filename = pstrdup(config_file);+ item->sourceline = ConfigFileLineno - 1;+ item->ignore = false;+ item->applied = false;+ item->next = NULL;+ if (*head_p == NULL)+ *head_p = item;+ else+ (*tail_p)->next = item;+ *tail_p = item;+ }++ /* break out of loop if read EOF, else loop for next line */+ if (token == 0)+ break;+ continue;++parse_error:+ /* release storage if we allocated any on this line */+ if (opt_name)+ pfree(opt_name);+ if (opt_value)+ pfree(opt_value);++ /* report the error */+ if (token == GUC_EOL || token == 0)+ {+ pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near end of line\n",+ config_file, ConfigFileLineno - 1);+ record_config_file_error("syntax error",+ config_file, ConfigFileLineno - 1,+ head_p, tail_p);+ }+ else+ {+ pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near token \"%s\"\n",+ config_file, ConfigFileLineno, yytext);+ record_config_file_error("syntax error",+ config_file, ConfigFileLineno,+ head_p, tail_p);+ }+ OK = false;+ errorcount++;++ /*+ * To avoid producing too much noise when fed a totally bogus file,+ * give up after 100 syntax errors per file (an arbitrary number).+ */+ if (errorcount >= 100)+ {+ pg_log(PG_WARNING, "too many syntax errors found, abandoning file \"%s\"\n",+ config_file);+ break;+ }++ /* resync to next end-of-line or EOF */+ while (token != GUC_EOL && token != 0)+ token = yylex();+ /* break out of loop on EOF */+ if (token == 0)+ break;+ }++cleanup:+ yy_delete_buffer(lex_buffer);+ /* Each recursion level must save and restore these static variables. */+ ConfigFileLineno = save_ConfigFileLineno;+ GUC_flex_fatal_jmp = save_GUC_flex_fatal_jmp;+ return OK;+}++/*+ * Read and parse all config files in a subdirectory in alphabetical order+ *+ * includedir is the absolute or relative path to the subdirectory to scan.+ *+ * calling_file/calling_lineno identify the source of the request.+ * Pass NULL/0 if not recursing from an inclusion request.+ *+ * See ParseConfigFp for further details.+ */+bool+ParseConfigDirectory(const char *includedir,+ const char *calling_file, int calling_lineno,+ int depth, int elevel,+ ConfigVariable **head_p,+ ConfigVariable **tail_p)+{+ char *directory;+ DIR *d;+ struct dirent *de;+ char **filenames;+ int num_filenames;+ int size_filenames;+ bool status;++ directory = AbsoluteConfigLocation(includedir, calling_file);+ d = opendir(directory);+ if (d == NULL)+ {+ pg_log(PG_WARNING, "could not open configuration directory \"%s\"\n",+ directory);+ record_config_file_error(psprintf("could not open directory \"%s\"",+ directory),+ calling_file, calling_lineno,+ head_p, tail_p);+ status = false;+ goto cleanup;+ }++ /*+ * Read the directory and put the filenames in an array, so we can sort+ * them prior to processing the contents.+ */+ size_filenames = 32;+ filenames = (char **) palloc(size_filenames * sizeof(char *));+ num_filenames = 0;++ while ((de = readdir(d)) != NULL)+ {+ struct stat st;+ char filename[MAXPGPATH];++ /*+ * Only parse files with names ending in ".conf". Explicitly reject+ * files starting with ".". This excludes things like "." and "..",+ * as well as typical hidden files, backup files, and editor debris.+ */+ if (strlen(de->d_name) < 6)+ continue;+ if (de->d_name[0] == '.')+ continue;+ if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0)+ continue;++ join_path_components(filename, directory, de->d_name);+ canonicalize_path(filename);+ if (stat(filename, &st) == 0)+ {+ if (!S_ISDIR(st.st_mode))+ {+ /* Add file to array, increasing its size in blocks of 32 */+ if (num_filenames >= size_filenames)+ {+ size_filenames += 32;+ filenames = (char **) repalloc(filenames,+ size_filenames * sizeof(char *));+ }+ filenames[num_filenames] = pstrdup(filename);+ num_filenames++;+ }+ }+ else+ {+ /*+ * stat does not care about permissions, so the most likely reason+ * a file can't be accessed now is if it was removed between the+ * directory listing and now.+ */+ pg_log(PG_WARNING, "could not stat file \"%s\"\n", filename);+ record_config_file_error(psprintf("could not stat file \"%s\"",+ filename),+ calling_file, calling_lineno,+ head_p, tail_p);+ status = false;+ goto cleanup;+ }+ }++ if (num_filenames > 0)+ {+ int i;++ qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp);+ for (i = 0; i < num_filenames; i++)+ {+ if (!ParseConfigFile(filenames[i], true,+ calling_file, calling_lineno,+ depth, elevel,+ head_p, tail_p))+ {+ status = false;+ goto cleanup;+ }+ }+ }+ status = true;++cleanup:+ if (d)+ closedir(d);+ pfree(directory);+ return status;+}++/*+ * Free a list of ConfigVariables, including the names and the values+ */+void+FreeConfigVariables(ConfigVariable *list)+{+ ConfigVariable *item;++ item = list;+ while (item)+ {+ ConfigVariable *next = item->next;++ FreeConfigVariable(item);+ item = next;+ }+}++/*+ * Free a single ConfigVariable+ */+static void+FreeConfigVariable(ConfigVariable *item)+{+ if (item->name)+ pfree(item->name);+ if (item->value)+ pfree(item->value);+ if (item->errmsg)+ pfree(item->errmsg);+ if (item->filename)+ pfree(item->filename);+ pfree(item);+}+++/*+ * scanstr+ *+ * Strip the quotes surrounding the given string, and collapse any embedded+ * '' sequences and backslash escapes.+ *+ * the string returned is palloc'd and should eventually be pfree'd by the+ * caller.+ */+static char *+GUC_scanstr(const char *s)+{+ char *newStr;+ int len,+ i,+ j;++ Assert(s != NULL && s[0] == '\'');+ len = strlen(s);+ Assert(len >= 2);+ Assert(s[len - 1] == '\'');++ /* Skip the leading quote; we'll handle the trailing quote below */+ s++, len--;++ /* Since len still includes trailing quote, this is enough space */+ newStr = palloc(len);++ for (i = 0, j = 0; i < len; i++)+ {+ if (s[i] == '\\')+ {+ i++;+ switch (s[i])+ {+ case 'b':+ newStr[j] = '\b';+ break;+ case 'f':+ newStr[j] = '\f';+ break;+ case 'n':+ newStr[j] = '\n';+ break;+ case 'r':+ newStr[j] = '\r';+ break;+ case 't':+ newStr[j] = '\t';+ break;+ case '0':+ case '1':+ case '2':+ case '3':+ case '4':+ case '5':+ case '6':+ case '7':+ {+ int k;+ long octVal = 0;++ for (k = 0;+ s[i + k] >= '0' && s[i + k] <= '7' && k < 3;+ k++)+ octVal = (octVal << 3) + (s[i + k] - '0');+ i += k - 1;+ newStr[j] = ((char) octVal);+ }+ break;+ default:+ newStr[j] = s[i];+ break;+ } /* switch */+ }+ else if (s[i] == '\'' && s[i + 1] == '\'')+ {+ /* doubled quote becomes just one quote */+ newStr[j] = s[++i];+ }+ else+ newStr[j] = s[i];+ j++;+ }++ /* We copied the ending quote to newStr, so replace with \0 */+ Assert(j > 0 && j <= len);+ newStr[--j] = '\0';++ return newStr;+}diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.cindex 40028471bf..11a9c26cd2 100644--- a/src/bin/pg_rewind/parsexlog.c+++ b/src/bin/pg_rewind/parsexlog.c@@ -12,6 +12,7 @@
#include "postgres_fe.h"
#include <unistd.h>
+#include <sys/stat.h>
#include "pg_rewind.h"
#include "filemap.h"
@@ -45,7 +46,10 @@ static char xlogfpath[MAXPGPATH];
typedef struct XLogPageReadPrivate
{
const char *datadir;
+ const char *restoreCommand;
int tliIndex;
+ XLogRecPtr oldrecptr;+ TimeLineID oldtli;
} XLogPageReadPrivate;
static int SimpleXLogPageRead(XLogReaderState *xlogreader,
@@ -53,6 +57,10 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
+static bool RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand,+ const char *lastRestartPointFname);+
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
* index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
@@ -60,15 +68,19 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
*/
void
extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
- XLogRecPtr endpoint)+ ControlFileData *targetCF, const char *restore_command)
{
XLogRecord *record;
+ XLogRecPtr endpoint = targetCF->checkPoint;
XLogReaderState *xlogreader;
char *errormsg;
XLogPageReadPrivate private;
private.datadir = datadir;
private.tliIndex = tliIndex;
+ private.restoreCommand = restore_command;+ private.oldrecptr = targetCF->checkPointCopy.redo;+ private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -154,9 +166,9 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
* Find the previous checkpoint preceding given WAL location.
*/
void
-findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,+findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo)+ XLogRecPtr *lastchkptredo, const char *restoreCommand)
{
/* Walk backwards, starting from the given record */
XLogRecord *record;
@@ -181,6 +193,9 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
private.datadir = datadir;
private.tliIndex = tliIndex;
+ private.restoreCommand = restoreCommand;+ private.oldrecptr = targetCF->checkPointCopy.redo;+ private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -291,9 +306,53 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
- printf(_("could not open file \"%s\": %s\n"), xlogfpath,+ bool restore_ok;+ char lastRestartPointFname[MAXFNAMELEN];+ XLogSegNo restartSegNo;++ /*+ * If we have no restore_command to execute, then exit.+ */+ if (private->restoreCommand == NULL)+ {+ printf(_("could not open file \"%s\": %s\n"), xlogfpath,
strerror(errno));
- return -1;+ return -1;+ }+ + XLByteToSeg(private->oldrecptr, restartSegNo, WalSegSz);+ XLogFileName(lastRestartPointFname, private->oldtli, restartSegNo,+ WalSegSz);++ /*+ * Since we have restore_command to execute, then try to retreive+ * missing WAL file from the archive.+ */+ restore_ok = RestoreArchivedWAL(private->datadir,+ xlogfname,+ WalSegSz,+ private->restoreCommand,+ lastRestartPointFname);++ if (restore_ok)+ {+ xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);++ if (xlogreadfd < 0)+ {+ printf(_("could not open restored from archive file \"%s\": %s\n"), xlogfpath,+ strerror(errno));+ return -1;+ }+ else+ pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", xlogfpath);+ }+ else+ {+ printf(_("could not restore file \"%s\" from archive: %s\n"), xlogfname,+ strerror(errno));+ return -1;+ }
}
}
@@ -409,3 +468,116 @@ extractPageInfo(XLogReaderState *record)
process_block_change(forknum, rnode, blkno);
}
}
++/*+ * Attempt to retrieve the specified file from off-line archival storage.+ * If successful return true.+ *+ * For fixed-size files, the caller may pass the expected size as an+ * additional crosscheck on successful recovery. If the file size is not+ * known, set expectedSize = 0.+ * + * This is an adapted to frontend version of+ * RestoreArchivedFile function from transam/xlogarchive.c+ */+bool+RestoreArchivedWAL(const char *path, const char *xlogfname, off_t expectedSize,+ const char *restoreCommand, const char *lastRestartPointFname)+{+ char xlogpath[MAXPGPATH];+ char xlogRestoreCmd[MAXPGPATH];+ char *dp;+ char *endp;+ const char *sp;+ int rc;+ struct stat stat_buf;++ snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);++ /*+ * Construct the command to be executed.+ */+ dp = xlogRestoreCmd;+ endp = xlogRestoreCmd + MAXPGPATH - 1;+ *endp = '\0';++ for (sp = restoreCommand; *sp; sp++)+ {+ if (*sp == '%')+ {+ switch (sp[1])+ {+ case 'p':+ /* %p: relative path of target file */+ sp++;+ StrNCpy(dp, xlogpath, endp - dp);+ make_native_path(dp);+ dp += strlen(dp);+ break;+ case 'f':+ /* %f: filename of desired file */+ sp++;+ StrNCpy(dp, xlogfname, endp - dp);+ dp += strlen(dp);+ break;+ case 'r':+ /* %r: filename of last restartpoint */+ sp++;+ StrNCpy(dp, lastRestartPointFname, endp - dp);+ dp += strlen(dp);+ break;+ case '%':+ /* convert %% to a single % */+ sp++;+ if (dp < endp)+ *dp++ = *sp;+ break;+ default:+ /* otherwise treat the % as not special */+ if (dp < endp)+ *dp++ = *sp;+ break;+ }+ }+ else+ {+ if (dp < endp)+ *dp++ = *sp;+ }+ }+ *dp = '\0';++ /*+ * Execute restore_command, which should copy+ * the missing WAL file from archival storage.+ */+ rc = system(xlogRestoreCmd);++ if (rc == 0)+ {+ /*+ * Command apparently succeeded, but let's make sure the file is+ * really there now and has the correct size.+ */+ if (stat(xlogpath, &stat_buf) == 0)+ {+ if (expectedSize > 0 && stat_buf.st_size != expectedSize)+ {+ printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"),+ xlogfname, (unsigned long) stat_buf.st_size, (unsigned long) expectedSize,+ strerror(errno));+ }+ else+ return true;+ }+ else+ {+ /* Stat failed */+ printf(_("could not stat file \"%s\": %s"),+ xlogpath,+ strerror(errno));+ }+ }++ return false;+}diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.cindex 9653106386..816a91fd78 100644--- a/src/bin/pg_rewind/pg_rewind.c+++ b/src/bin/pg_rewind/pg_rewind.c@@ -19,11 +19,13 @@
#include "file_ops.h"
#include "filemap.h"
#include "logging.h"
+#include "guc-file-fe.h"
#include "access/timeline.h"
#include "access/xlog_internal.h"
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
+#include "common/controldata_utils.h"
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "common/restricted_token.h"
@@ -52,11 +54,13 @@ int WalSegSz;
char *datadir_target = NULL;
char *datadir_source = NULL;
char *connstr_source = NULL;
+char *restore_command = NULL;
bool debug = false;
bool showprogress = false;
bool dry_run = false;
bool do_sync = true;
+bool restore_wals = false;
/* Target history */
TimeLineHistoryEntry *targetHistory;
@@ -75,6 +79,9 @@ usage(const char *progname)
printf(_(" -N, --no-sync do not wait for changes to be written\n"));
printf(_(" safely to disk\n"));
printf(_(" -P, --progress write progress messages\n"));
+ printf(_(" -r, --use-postgresql-conf use restore_command in the postgresql.conf to\n"));+ printf(_(" retreive WALs from archive\n"));+ printf(_(" -R, --restore-command=COMMAND restore command\n"));
printf(_(" --debug write a lot of debug messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -94,9 +101,12 @@ main(int argc, char **argv)
{"dry-run", no_argument, NULL, 'n'},
{"no-sync", no_argument, NULL, 'N'},
{"progress", no_argument, NULL, 'P'},
+ {"use-postgresql-conf", no_argument, NULL, 'r'},+ {"restore-command", required_argument, NULL, 'R'},
{"debug", no_argument, NULL, 3},
{NULL, 0, NULL, 0}
};
+ char recfile_fullpath[MAXPGPATH];
int option_index;
int c;
XLogRecPtr divergerec;
@@ -129,7 +139,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1)+ while ((c = getopt_long(argc, argv, "DR:nNPr", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -140,6 +150,10 @@ main(int argc, char **argv)
case 'P':
showprogress = true;
break;
+ + case 'r':+ restore_wals = true;+ break;
case 'n':
dry_run = true;
@@ -157,6 +171,10 @@ main(int argc, char **argv)
datadir_target = pg_strdup(optarg);
break;
+ case 'R':+ restore_command = pg_strdup(optarg);+ break;+
case 1: /* --source-pgdata */
datadir_source = pg_strdup(optarg);
break;
@@ -223,6 +241,71 @@ main(int argc, char **argv)
umask(pg_mode_mask);
+ if (restore_command != NULL)+ {+ pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command);+ }+ else if (restore_wals)+ {+ FILE *recovery_conf_file;++ /* + * Look for recovery.conf in the target data directory and+ * try to get restore_command from there.+ */+ snprintf(recfile_fullpath, sizeof(recfile_fullpath), "%s/%s", datadir_target, RECOVERY_COMMAND_FILE);+ recovery_conf_file = fopen(recfile_fullpath, "r");++ if (recovery_conf_file == NULL)+ {+ fprintf(stderr, _("%s: option -r/--use-postgresql-conf is specified, but postgreslq.conf is absent in the target directory\n"),+ progname);+ fprintf(stdout, _("You have to add postgresql.conf or pass restore_command with -R/--restore-command option.\n"));+ exit(1);+ }+ else+ {+ ConfigVariable *item,+ *head = NULL,+ *tail = NULL;+ bool config_is_parsed;++ /*+ * We pass a fullpath to the recovery.conf as calling_file here, since+ * parser will use its parent directory as base for all further includes+ * if any exist.+ */+ config_is_parsed = ParseConfigFile(RECOVERY_COMMAND_FILE, true,+ recfile_fullpath, 0, 0,+ PG_WARNING, &head, &tail);+ fclose(recovery_conf_file);++ if (config_is_parsed)+ {+ for (item = head; item; item = item->next)+ {+ if (strcmp(item->name, "restore_command") == 0)+ {+ if (restore_command != NULL)+ {+ pfree(restore_command);+ restore_command = NULL;+ }+ restore_command = pstrdup(item->value);+ pg_log(PG_DEBUG, "using restore_command=\'%s\' from recovery.conf.\n", restore_command);+ }+ }++ if (restore_command == NULL)+ pg_fatal("could not find restore_command in recovery.conf file %s\n", recfile_fullpath);+ }+ else+ pg_fatal("could not parse recovery.conf file %s\n", recfile_fullpath);++ FreeConfigVariables(head);+ }+ }+
/* Connect to remote server */
if (connstr_source)
libpqConnect(connstr_source);
@@ -294,9 +377,9 @@ main(int argc, char **argv)
exit(0);
}
- findLastCheckpoint(datadir_target, divergerec,+ findLastCheckpoint(datadir_target, &ControlFile_target, divergerec,
lastcommontliIndex,
- &chkptrec, &chkpttli, &chkptredo);+ &chkptrec, &chkpttli, &chkptredo, restore_command);
printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"),
(uint32) (chkptrec >> 32), (uint32) chkptrec,
chkpttli);
@@ -319,7 +402,7 @@ main(int argc, char **argv)
*/
pg_log(PG_PROGRESS, "reading WAL in target\n");
extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
- ControlFile_target.checkPoint);+ &ControlFile_target, restore_command);
filemap_finalize();
if (showprogress)
diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.hindex 3f4ba7a267..ba9d13e9bd 100644--- a/src/bin/pg_rewind/pg_rewind.h+++ b/src/bin/pg_rewind/pg_rewind.h@@ -14,6 +14,7 @@
#include "datapagemap.h"
#include "access/timeline.h"
+#include "catalog/pg_control.h"
#include "storage/block.h"
#include "storage/relfilenode.h"
@@ -32,11 +33,10 @@ extern int targetNentries;
/* in parsexlog.c */
extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
- int tliIndex, XLogRecPtr endpoint);-extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,- int tliIndex,- XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,- XLogRecPtr *lastchkptredo);+ int tliIndex, ControlFileData *targetCF, const char *restoreCommand);+extern void findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr searchptr,+ int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,+ XLogRecPtr *lastchkptredo, const char *restoreCommand);
extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
int tliIndex);
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.plindex 53dbf45be2..22777bff82 100644--- a/src/bin/pg_rewind/t/001_basic.pl+++ b/src/bin/pg_rewind/t/001_basic.pl@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 10;+use Test::More tests => 15;
use RewindTest;
@@ -103,5 +103,6 @@ in master, before promotion
# Run the test in both modes
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.plindex 2c9e427831..10c835efc1 100644--- a/src/bin/pg_rewind/t/002_databases.pl+++ b/src/bin/pg_rewind/t/002_databases.pl@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 6;+use Test::More tests => 9;
use RewindTest;
@@ -59,5 +59,6 @@ template1
# Run the test in both modes.
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.plindex 496f38c457..f94ce329f4 100644--- a/src/bin/pg_rewind/t/003_extrafiles.pl+++ b/src/bin/pg_rewind/t/003_extrafiles.pl@@ -3,7 +3,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 4;+use Test::More tests => 6;
use File::Find;
@@ -87,5 +87,6 @@ sub run_test
# Run the test in both modes.
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pmindex 2921d193a1..3da0fdc956 100644--- a/src/tools/msvc/Mkvcbuild.pm+++ b/src/tools/msvc/Mkvcbuild.pm@@ -73,6 +73,7 @@ my $frontend_extraincludes = {
};
my $frontend_extrasource = {
'psql' => ['src/bin/psql/psqlscanslash.l'],
+ 'pg_rewind' => ['src/bin/pg_rewind/guc-file-fe.l'],
'pgbench' =>
[ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ]
};
--
2.17.1
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: a.kondratov@postgrespro.ru, 9erthalion6@gmail.com, michael@paquier.xyz, alvherre@2ndquadrant.com, vladimirlesk@yandex-team.ru, dsarafan@yandex-team.ru
Subject: Re: [Patch] pg_rewind: options to use restore_command from recovery.conf or command line
In-Reply-To: <13ec67dc-2c6b-777e-902a-1141fdb8302e@postgrespro.ru>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox