From: Alexey Kondratov Date: Fri, 21 Dec 2018 14:00:30 +0300 Subject: [PATCH] pg_rewind: options to use restore_command from postgresql.conf or command line. --- src/bin/pg_rewind/Makefile | 5 +- src/bin/pg_rewind/RewindTest.pm | 46 +- src/bin/pg_rewind/guc-file-fe.h | 40 ++ src/bin/pg_rewind/guc-file-fe.l | 776 ++++++++++++++++++++++++++ src/bin/pg_rewind/parsexlog.c | 182 +++++- src/bin/pg_rewind/pg_rewind.c | 91 ++- src/bin/pg_rewind/pg_rewind.h | 10 +- src/bin/pg_rewind/t/001_basic.pl | 3 +- src/bin/pg_rewind/t/002_databases.pl | 3 +- src/bin/pg_rewind/t/003_extrafiles.pl | 3 +- src/tools/msvc/Mkvcbuild.pm | 1 + 11 files changed, 1141 insertions(+), 19 deletions(-) create mode 100644 src/bin/pg_rewind/guc-file-fe.h create mode 100644 src/bin/pg_rewind/guc-file-fe.l diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile index 2bcfcc61af..a0f5f97544 100644 --- a/src/bin/pg_rewind/Makefile +++ b/src/bin/pg_rewind/Makefile @@ -15,11 +15,12 @@ subdir = src/bin/pg_rewind top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) +override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) LDFLAGS_INTERNAL += $(libpq_pgport) OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \ fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o \ + guc-file-fe.o \ $(WIN32RES) EXTRA_CLEAN = xlogreader.c @@ -32,6 +33,8 @@ pg_rewind: $(OBJS) | submake-libpq submake-libpgport xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% rm -f $@ && $(LN_S) $< . +distprep: guc-file-fe.c + install: all installdirs $(INSTALL_PROGRAM) pg_rewind$(X) '$(DESTDIR)$(bindir)/pg_rewind$(X)' diff --git a/src/bin/pg_rewind/RewindTest.pm b/src/bin/pg_rewind/RewindTest.pm index 3d07da5d94..b43c18a8c3 100644 --- a/src/bin/pg_rewind/RewindTest.pm +++ b/src/bin/pg_rewind/RewindTest.pm @@ -39,7 +39,9 @@ use Carp; use Config; use Exporter 'import'; use File::Copy; -use File::Path qw(rmtree); +use File::Glob ':bsd_glob'; +use File::Path qw(remove_tree make_path); +use File::Spec::Functions 'catpath'; use IPC::Run qw(run); use PostgresNode; use TestLib; @@ -250,6 +252,48 @@ sub run_pg_rewind ], 'pg_rewind remote'); } + elsif ($test_mode eq "archive") + { + + # Do rewind using a local pgdata as source and + # specified directory with target WALs archive. + my $wals_archive_dir = catpath(${TestLib::tmp_check}, 'master_wals_archive'); + my $test_master_datadir = $node_master->data_dir; + my @wal_files = bsd_glob catpath($test_master_datadir, 'pg_wal', '0000000*'); + my $restore_command; + + remove_tree($wals_archive_dir); + make_path($wals_archive_dir) or die; + + # Move all old master WAL files to the archive. + # Old master should be stopped at this point. + foreach my $wal_file (@wal_files) + { + move($wal_file, "$wals_archive_dir/") or die; + } + + if ($windows_os) + { + $restore_command = "copy $wals_archive_dir\\\%f \%p"; + } + else + { + $restore_command = "cp $wals_archive_dir/\%f \%p"; + } + + # Stop the new master and be ready to perform the rewind. + $node_standby->stop; + command_ok( + [ + 'pg_rewind', + "--debug", + "--source-pgdata=$standby_pgdata", + "--target-pgdata=$master_pgdata", + "--no-sync", + "-R", $restore_command + ], + 'pg_rewind archive'); + } else { diff --git a/src/bin/pg_rewind/guc-file-fe.h b/src/bin/pg_rewind/guc-file-fe.h new file mode 100644 index 0000000000..3a078c5cdd --- /dev/null +++ b/src/bin/pg_rewind/guc-file-fe.h @@ -0,0 +1,40 @@ +#ifndef PG_REWIND_GUC_FILE_FE_H +#define PG_REWIND_GUC_FILE_FE_H + +#include "c.h" + +#define RECOVERY_COMMAND_FILE "postgresql.conf" + +/* + * Parsing the configuration file(s) will return a list of name-value pairs + * with source location info. We also abuse this data structure to carry + * error reports about the config files. An entry reporting an error will + * have errmsg != NULL, and might have NULLs for name, value, and/or filename. + * + * If "ignore" is true, don't attempt to apply the item (it might be an error + * report, or an item we determined to be duplicate). "applied" is set true + * if we successfully applied, or could have applied, the setting. + */ +typedef struct ConfigVariable +{ + char *name; + char *value; + char *errmsg; + char *filename; + int sourceline; + bool ignore; + bool applied; + struct ConfigVariable *next; +} ConfigVariable; + +extern bool ParseConfigFile(const char *config_file, bool strict, + const char *calling_file, int calling_lineno, + int depth, int elevel, + ConfigVariable **head_p, ConfigVariable **tail_p); + +extern bool ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, + ConfigVariable **head_p, ConfigVariable **tail_p); + +extern void FreeConfigVariables(ConfigVariable *list); + +#endif /* PG_REWIND_GUC_FILE_FE_H */ diff --git a/src/bin/pg_rewind/guc-file-fe.l b/src/bin/pg_rewind/guc-file-fe.l new file mode 100644 index 0000000000..b7ef47111a --- /dev/null +++ b/src/bin/pg_rewind/guc-file-fe.l @@ -0,0 +1,776 @@ +/* -*-pgsql-c-*- */ +/* + * Configuration files parser for usage in frontend. + * + * Copyright (c) 2000-2018, PostgreSQL Global Development Group + * + * src/bin/pg_rewind/guc-file-fe.l + * Modified version of src/backend/utils/misc/guc-file.l + */ + +%{ + +#include "postgres_fe.h" + +#include +#include +#include +#include +#include +#include + +#include "guc-file-fe.h" +#include "logging.h" + +#include "common/fe_memutils.h" + +/* + * flex emits a yy_fatal_error() function that it calls in response to + * critical errors like malloc failure, file I/O errors, and detection of + * internal inconsistency. That function prints a message and calls exit(). + * Mutate it to instead call our handler, which jumps out of the parser. + */ +#undef fprintf +#define fprintf(file, fmt, msg) GUC_flex_fatal(msg) + +enum +{ + GUC_ID = 1, + GUC_STRING = 2, + GUC_INTEGER = 3, + GUC_REAL = 4, + GUC_EQUALS = 5, + GUC_UNQUOTED_STRING = 6, + GUC_QUALIFIED_ID = 7, + GUC_EOL = 99, + GUC_ERROR = 100 +}; + +static unsigned int ConfigFileLineno; +static const char *GUC_flex_fatal_errmsg; +static sigjmp_buf *GUC_flex_fatal_jmp; + +static void FreeConfigVariable(ConfigVariable *item); + +static void record_config_file_error(const char *errmsg, + const char *config_file, + int lineno, + ConfigVariable **head_p, + ConfigVariable **tail_p); + +static int GUC_flex_fatal(const char *msg); +static char *GUC_scanstr(const char *s); +static bool ParseConfigDirectory(const char *includedir, + const char *calling_file, int calling_lineno, + int depth, int elevel, + ConfigVariable **head_p, + ConfigVariable **tail_p); + +/* LCOV_EXCL_START */ + +%} + +%option 8bit +%option never-interactive +%option nodefault +%option noinput +%option nounput +%option noyywrap +%option warn +%option prefix="GUC_yy" + + +SIGN ("-"|"+") +DIGIT [0-9] +HEXDIGIT [0-9a-fA-F] + +UNIT_LETTER [a-zA-Z] + +INTEGER {SIGN}?({DIGIT}+|0x{HEXDIGIT}+){UNIT_LETTER}* + +EXPONENT [Ee]{SIGN}?{DIGIT}+ +REAL {SIGN}?{DIGIT}*"."{DIGIT}*{EXPONENT}? + +LETTER [A-Za-z_\200-\377] +LETTER_OR_DIGIT [A-Za-z_0-9\200-\377] + +ID {LETTER}{LETTER_OR_DIGIT}* +QUALIFIED_ID {ID}"."{ID} + +UNQUOTED_STRING {LETTER}({LETTER_OR_DIGIT}|[-._:/])* +STRING \'([^'\\\n]|\\.|\'\')*\' + +%% + +\n ConfigFileLineno++; return GUC_EOL; +[ \t\r]+ /* eat whitespace */ +#.* /* eat comment (.* matches anything until newline) */ + +{ID} return GUC_ID; +{QUALIFIED_ID} return GUC_QUALIFIED_ID; +{STRING} return GUC_STRING; +{UNQUOTED_STRING} return GUC_UNQUOTED_STRING; +{INTEGER} return GUC_INTEGER; +{REAL} return GUC_REAL; += return GUC_EQUALS; + +. return GUC_ERROR; + +%% + +/* LCOV_EXCL_STOP */ + +/* + * the bare comparison function for GUC names + */ +static int +guc_name_compare(const char *namea, const char *nameb) +{ + /* + * The temptation to use strcasecmp() here must be resisted, because the + * array ordering has to remain stable across setlocale() calls. So, build + * our own with a simple ASCII-only downcasing. + */ + while (*namea && *nameb) + { + char cha = *namea++; + char chb = *nameb++; + + if (cha >= 'A' && cha <= 'Z') + cha += 'a' - 'A'; + if (chb >= 'A' && chb <= 'Z') + chb += 'a' - 'A'; + if (cha != chb) + return cha - chb; + } + if (*namea) + return 1; /* a is longer */ + if (*nameb) + return -1; /* b is longer */ + return 0; +} + +/* + * Given a configuration file or directory location that may be a relative + * path, return an absolute one. We consider the location to be relative to + * the directory holding the calling file. + */ +static char * +AbsoluteConfigLocation(const char *location, const char *calling_file) +{ + char abs_path[MAXPGPATH]; + + if (is_absolute_path(location)) + return pstrdup(location); + else + { + strlcpy(abs_path, calling_file, sizeof(abs_path)); + get_parent_directory(abs_path); + join_path_components(abs_path, abs_path, location); + canonicalize_path(abs_path); + + return pstrdup(abs_path); + } +} + +/* + * Read and parse a single configuration file. This function recurses + * to handle "include" directives. + * + * If "strict" is true, treat failure to open the config file as an error, + * otherwise just skip the file. + * + * calling_file/calling_lineno identify the source of the request. + * Pass NULL/0 if not recursing from an inclusion request. + * + * See ParseConfigFp for further details. This one merely adds opening the + * config file rather than working from a caller-supplied file descriptor, + * and absolute-ifying the path name if necessary. + */ +bool +ParseConfigFile(const char *config_file, bool strict, + const char *calling_file, int calling_lineno, + int depth, int elevel, + ConfigVariable **head_p, + ConfigVariable **tail_p) +{ + char *abs_path; + bool OK = true; + FILE *fp; + + /* + * Reject too-deep include nesting depth. This is just a safety check to + * avoid dumping core due to stack overflow if an include file loops back + * to itself. The maximum nesting depth is pretty arbitrary. + */ + if (depth > 10) + { + pg_log(PG_WARNING, + "could not open configuration file \"%s\": maximum nesting depth exceeded\n", + config_file); + record_config_file_error("nesting depth exceeded", + calling_file, calling_lineno, + head_p, tail_p); + return false; + } + + abs_path = AbsoluteConfigLocation(config_file, calling_file); + fp = fopen(abs_path, "r"); + if (!fp) + { + if (strict) + { + pg_log(PG_WARNING, + "could not open configuration file \"%s\"\n", + abs_path); + record_config_file_error(psprintf("could not open file \"%s\"", + abs_path), + calling_file, calling_lineno, + head_p, tail_p); + OK = false; + } + else + { + pg_log(PG_WARNING, + "skipping missing configuration file \"%s\"\n", + abs_path); + } + goto cleanup; + } + + OK = ParseConfigFp(fp, abs_path, depth, elevel, head_p, tail_p); + +cleanup: + if (fp) + fclose(fp); + pfree(abs_path); + + return OK; +} + +/* + * Capture an error message in the ConfigVariable list returned by + * config file parsing. + */ +static void +record_config_file_error(const char *errmsg, + const char *config_file, + int lineno, + ConfigVariable **head_p, + ConfigVariable **tail_p) +{ + ConfigVariable *item; + + item = palloc(sizeof *item); + item->name = NULL; + item->value = NULL; + item->errmsg = pstrdup(errmsg); + item->filename = config_file ? pstrdup(config_file) : NULL; + item->sourceline = lineno; + item->ignore = true; + item->applied = false; + item->next = NULL; + if (*head_p == NULL) + *head_p = item; + else + (*tail_p)->next = item; + *tail_p = item; +} + +/* + * Flex fatal errors bring us here. Stash the error message and jump back to + * ParseConfigFp(). Assume all msg arguments point to string constants; this + * holds for flex 2.5.31 (earliest we support) and flex 2.5.35 (latest as of + * this writing). Otherwise, we would need to copy the message. + * + * We return "int" since this takes the place of calls to fprintf(). +*/ +static int +GUC_flex_fatal(const char *msg) +{ + GUC_flex_fatal_errmsg = msg; + siglongjmp(*GUC_flex_fatal_jmp, 1); + return 0; /* keep compiler quiet */ +} + +/* + * Read and parse a single configuration file. This function recurses + * to handle "include" directives. + * + * Input parameters: + * fp: file pointer from AllocateFile for the configuration file to parse + * config_file: absolute or relative path name of the configuration file + * depth: recursion depth (should be 0 in the outermost call) + * elevel: error logging level to use + * Input/Output parameters: + * head_p, tail_p: head and tail of linked list of name/value pairs + * + * *head_p and *tail_p must be initialized, either to NULL or valid pointers + * to a ConfigVariable list, before calling the outer recursion level. Any + * name-value pairs read from the input file(s) will be appended to the list. + * Error reports will also be appended to the list, if elevel < ERROR. + * + * Returns TRUE if successful, FALSE if an error occurred. + * + * Note: this function is used to parse not only postgresql.conf, but + * various other configuration files that use the same "name = value" + * syntax. Hence, do not do anything here or in the subsidiary routines + * ParseConfigFile/ParseConfigDirectory that assumes we are processing + * GUCs specifically. + */ +bool +ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel, + ConfigVariable **head_p, ConfigVariable **tail_p) +{ + volatile bool OK = true; + unsigned int save_ConfigFileLineno = ConfigFileLineno; + sigjmp_buf *save_GUC_flex_fatal_jmp = GUC_flex_fatal_jmp; + sigjmp_buf flex_fatal_jmp; + volatile YY_BUFFER_STATE lex_buffer = NULL; + int errorcount; + int token; + + if (sigsetjmp(flex_fatal_jmp, 1) == 0) + GUC_flex_fatal_jmp = &flex_fatal_jmp; + else + { + /* + * Regain control after a fatal, internal flex error. It may have + * corrupted parser state. Consequently, abandon the file, but trust + * that the state remains sane enough for yy_delete_buffer(). + */ + pg_log(PG_WARNING, "%s at file \"%s\" line %u\n", GUC_flex_fatal_errmsg, + config_file, ConfigFileLineno); + record_config_file_error(GUC_flex_fatal_errmsg, + config_file, ConfigFileLineno, + head_p, tail_p); + OK = false; + goto cleanup; + } + + /* + * Parse + */ + ConfigFileLineno = 1; + errorcount = 0; + + lex_buffer = yy_create_buffer(fp, YY_BUF_SIZE); + yy_switch_to_buffer(lex_buffer); + + /* This loop iterates once per logical line */ + while ((token = yylex())) + { + char *opt_name = NULL; + char *opt_value = NULL; + ConfigVariable *item; + + if (token == GUC_EOL) /* empty or comment line */ + continue; + + /* first token on line is option name */ + if (token != GUC_ID && token != GUC_QUALIFIED_ID) + goto parse_error; + opt_name = pstrdup(yytext); + + /* next we have an optional equal sign; discard if present */ + token = yylex(); + if (token == GUC_EQUALS) + token = yylex(); + + /* now we must have the option value */ + if (token != GUC_ID && + token != GUC_STRING && + token != GUC_INTEGER && + token != GUC_REAL && + token != GUC_UNQUOTED_STRING) + goto parse_error; + if (token == GUC_STRING) /* strip quotes and escapes */ + opt_value = GUC_scanstr(yytext); + else + opt_value = pstrdup(yytext); + + /* now we'd like an end of line, or possibly EOF */ + token = yylex(); + if (token != GUC_EOL) + { + if (token != 0) + goto parse_error; + /* treat EOF like \n for line numbering purposes, cf bug 4752 */ + ConfigFileLineno++; + } + + /* OK, process the option name and value */ + if (guc_name_compare(opt_name, "include_dir") == 0) + { + /* + * An include_dir directive isn't a variable and should be + * processed immediately. + */ + if (!ParseConfigDirectory(opt_value, + config_file, ConfigFileLineno - 1, + depth + 1, elevel, + head_p, tail_p)) + OK = false; + yy_switch_to_buffer(lex_buffer); + pfree(opt_name); + pfree(opt_value); + } + else if (guc_name_compare(opt_name, "include_if_exists") == 0) + { + /* + * An include_if_exists directive isn't a variable and should be + * processed immediately. + */ + if (!ParseConfigFile(opt_value, false, + config_file, ConfigFileLineno - 1, + depth + 1, elevel, + head_p, tail_p)) + OK = false; + yy_switch_to_buffer(lex_buffer); + pfree(opt_name); + pfree(opt_value); + } + else if (guc_name_compare(opt_name, "include") == 0) + { + /* + * An include directive isn't a variable and should be processed + * immediately. + */ + if (!ParseConfigFile(opt_value, true, + config_file, ConfigFileLineno - 1, + depth + 1, elevel, + head_p, tail_p)) + OK = false; + yy_switch_to_buffer(lex_buffer); + pfree(opt_name); + pfree(opt_value); + } + else + { + /* ordinary variable, append to list */ + item = palloc(sizeof *item); + item->name = opt_name; + item->value = opt_value; + item->errmsg = NULL; + item->filename = pstrdup(config_file); + item->sourceline = ConfigFileLineno - 1; + item->ignore = false; + item->applied = false; + item->next = NULL; + if (*head_p == NULL) + *head_p = item; + else + (*tail_p)->next = item; + *tail_p = item; + } + + /* break out of loop if read EOF, else loop for next line */ + if (token == 0) + break; + continue; + +parse_error: + /* release storage if we allocated any on this line */ + if (opt_name) + pfree(opt_name); + if (opt_value) + pfree(opt_value); + + /* report the error */ + if (token == GUC_EOL || token == 0) + { + pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near end of line\n", + config_file, ConfigFileLineno - 1); + record_config_file_error("syntax error", + config_file, ConfigFileLineno - 1, + head_p, tail_p); + } + else + { + pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near token \"%s\"\n", + config_file, ConfigFileLineno, yytext); + record_config_file_error("syntax error", + config_file, ConfigFileLineno, + head_p, tail_p); + } + OK = false; + errorcount++; + + /* + * To avoid producing too much noise when fed a totally bogus file, + * give up after 100 syntax errors per file (an arbitrary number). + */ + if (errorcount >= 100) + { + pg_log(PG_WARNING, "too many syntax errors found, abandoning file \"%s\"\n", + config_file); + break; + } + + /* resync to next end-of-line or EOF */ + while (token != GUC_EOL && token != 0) + token = yylex(); + /* break out of loop on EOF */ + if (token == 0) + break; + } + +cleanup: + yy_delete_buffer(lex_buffer); + /* Each recursion level must save and restore these static variables. */ + ConfigFileLineno = save_ConfigFileLineno; + GUC_flex_fatal_jmp = save_GUC_flex_fatal_jmp; + return OK; +} + +/* + * Read and parse all config files in a subdirectory in alphabetical order + * + * includedir is the absolute or relative path to the subdirectory to scan. + * + * calling_file/calling_lineno identify the source of the request. + * Pass NULL/0 if not recursing from an inclusion request. + * + * See ParseConfigFp for further details. + */ +bool +ParseConfigDirectory(const char *includedir, + const char *calling_file, int calling_lineno, + int depth, int elevel, + ConfigVariable **head_p, + ConfigVariable **tail_p) +{ + char *directory; + DIR *d; + struct dirent *de; + char **filenames; + int num_filenames; + int size_filenames; + bool status; + + directory = AbsoluteConfigLocation(includedir, calling_file); + d = opendir(directory); + if (d == NULL) + { + pg_log(PG_WARNING, "could not open configuration directory \"%s\"\n", + directory); + record_config_file_error(psprintf("could not open directory \"%s\"", + directory), + calling_file, calling_lineno, + head_p, tail_p); + status = false; + goto cleanup; + } + + /* + * Read the directory and put the filenames in an array, so we can sort + * them prior to processing the contents. + */ + size_filenames = 32; + filenames = (char **) palloc(size_filenames * sizeof(char *)); + num_filenames = 0; + + while ((de = readdir(d)) != NULL) + { + struct stat st; + char filename[MAXPGPATH]; + + /* + * Only parse files with names ending in ".conf". Explicitly reject + * files starting with ".". This excludes things like "." and "..", + * as well as typical hidden files, backup files, and editor debris. + */ + if (strlen(de->d_name) < 6) + continue; + if (de->d_name[0] == '.') + continue; + if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0) + continue; + + join_path_components(filename, directory, de->d_name); + canonicalize_path(filename); + if (stat(filename, &st) == 0) + { + if (!S_ISDIR(st.st_mode)) + { + /* Add file to array, increasing its size in blocks of 32 */ + if (num_filenames >= size_filenames) + { + size_filenames += 32; + filenames = (char **) repalloc(filenames, + size_filenames * sizeof(char *)); + } + filenames[num_filenames] = pstrdup(filename); + num_filenames++; + } + } + else + { + /* + * stat does not care about permissions, so the most likely reason + * a file can't be accessed now is if it was removed between the + * directory listing and now. + */ + pg_log(PG_WARNING, "could not stat file \"%s\"\n", filename); + record_config_file_error(psprintf("could not stat file \"%s\"", + filename), + calling_file, calling_lineno, + head_p, tail_p); + status = false; + goto cleanup; + } + } + + if (num_filenames > 0) + { + int i; + + qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp); + for (i = 0; i < num_filenames; i++) + { + if (!ParseConfigFile(filenames[i], true, + calling_file, calling_lineno, + depth, elevel, + head_p, tail_p)) + { + status = false; + goto cleanup; + } + } + } + status = true; + +cleanup: + if (d) + closedir(d); + pfree(directory); + return status; +} + +/* + * Free a list of ConfigVariables, including the names and the values + */ +void +FreeConfigVariables(ConfigVariable *list) +{ + ConfigVariable *item; + + item = list; + while (item) + { + ConfigVariable *next = item->next; + + FreeConfigVariable(item); + item = next; + } +} + +/* + * Free a single ConfigVariable + */ +static void +FreeConfigVariable(ConfigVariable *item) +{ + if (item->name) + pfree(item->name); + if (item->value) + pfree(item->value); + if (item->errmsg) + pfree(item->errmsg); + if (item->filename) + pfree(item->filename); + pfree(item); +} + + +/* + * scanstr + * + * Strip the quotes surrounding the given string, and collapse any embedded + * '' sequences and backslash escapes. + * + * the string returned is palloc'd and should eventually be pfree'd by the + * caller. + */ +static char * +GUC_scanstr(const char *s) +{ + char *newStr; + int len, + i, + j; + + Assert(s != NULL && s[0] == '\''); + len = strlen(s); + Assert(len >= 2); + Assert(s[len - 1] == '\''); + + /* Skip the leading quote; we'll handle the trailing quote below */ + s++, len--; + + /* Since len still includes trailing quote, this is enough space */ + newStr = palloc(len); + + for (i = 0, j = 0; i < len; i++) + { + if (s[i] == '\\') + { + i++; + switch (s[i]) + { + case 'b': + newStr[j] = '\b'; + break; + case 'f': + newStr[j] = '\f'; + break; + case 'n': + newStr[j] = '\n'; + break; + case 'r': + newStr[j] = '\r'; + break; + case 't': + newStr[j] = '\t'; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + int k; + long octVal = 0; + + for (k = 0; + s[i + k] >= '0' && s[i + k] <= '7' && k < 3; + k++) + octVal = (octVal << 3) + (s[i + k] - '0'); + i += k - 1; + newStr[j] = ((char) octVal); + } + break; + default: + newStr[j] = s[i]; + break; + } /* switch */ + } + else if (s[i] == '\'' && s[i + 1] == '\'') + { + /* doubled quote becomes just one quote */ + newStr[j] = s[++i]; + } + else + newStr[j] = s[i]; + j++; + } + + /* We copied the ending quote to newStr, so replace with \0 */ + Assert(j > 0 && j <= len); + newStr[--j] = '\0'; + + return newStr; +} diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 40028471bf..11a9c26cd2 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -12,6 +12,7 @@ #include "postgres_fe.h" #include +#include #include "pg_rewind.h" #include "filemap.h" @@ -45,7 +46,10 @@ static char xlogfpath[MAXPGPATH]; typedef struct XLogPageReadPrivate { const char *datadir; + const char *restoreCommand; int tliIndex; + XLogRecPtr oldrecptr; + TimeLineID oldtli; } XLogPageReadPrivate; static int SimpleXLogPageRead(XLogReaderState *xlogreader, @@ -53,6 +57,10 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, int reqLen, XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *pageTLI); +static bool RestoreArchivedWAL(const char *path, const char *xlogfname, + off_t expectedSize, const char *restoreCommand, + const char *lastRestartPointFname); + /* * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of @@ -60,15 +68,19 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, */ void extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex, - XLogRecPtr endpoint) + ControlFileData *targetCF, const char *restore_command) { XLogRecord *record; + XLogRecPtr endpoint = targetCF->checkPoint; XLogReaderState *xlogreader; char *errormsg; XLogPageReadPrivate private; private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restore_command; + private.oldrecptr = targetCF->checkPointCopy.redo; + private.oldtli = targetCF->checkPointCopy.ThisTimeLineID; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -154,9 +166,9 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex) * Find the previous checkpoint preceding given WAL location. */ void -findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, +findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo) + XLogRecPtr *lastchkptredo, const char *restoreCommand) { /* Walk backwards, starting from the given record */ XLogRecord *record; @@ -181,6 +193,9 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, private.datadir = datadir; private.tliIndex = tliIndex; + private.restoreCommand = restoreCommand; + private.oldrecptr = targetCF->checkPointCopy.redo; + private.oldtli = targetCF->checkPointCopy.ThisTimeLineID; xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead, &private); if (xlogreader == NULL) @@ -291,9 +306,53 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, if (xlogreadfd < 0) { - printf(_("could not open file \"%s\": %s\n"), xlogfpath, + bool restore_ok; + char lastRestartPointFname[MAXFNAMELEN]; + XLogSegNo restartSegNo; + + /* + * If we have no restore_command to execute, then exit. + */ + if (private->restoreCommand == NULL) + { + printf(_("could not open file \"%s\": %s\n"), xlogfpath, strerror(errno)); - return -1; + return -1; + } + + XLByteToSeg(private->oldrecptr, restartSegNo, WalSegSz); + XLogFileName(lastRestartPointFname, private->oldtli, restartSegNo, + WalSegSz); + + /* + * Since we have restore_command to execute, then try to retreive + * missing WAL file from the archive. + */ + restore_ok = RestoreArchivedWAL(private->datadir, + xlogfname, + WalSegSz, + private->restoreCommand, + lastRestartPointFname); + + if (restore_ok) + { + xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0); + + if (xlogreadfd < 0) + { + printf(_("could not open restored from archive file \"%s\": %s\n"), xlogfpath, + strerror(errno)); + return -1; + } + else + pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", xlogfpath); + } + else + { + printf(_("could not restore file \"%s\" from archive: %s\n"), xlogfname, + strerror(errno)); + return -1; + } } } @@ -409,3 +468,116 @@ extractPageInfo(XLogReaderState *record) process_block_change(forknum, rnode, blkno); } } + +/* + * Attempt to retrieve the specified file from off-line archival storage. + * If successful return true. + * + * For fixed-size files, the caller may pass the expected size as an + * additional crosscheck on successful recovery. If the file size is not + * known, set expectedSize = 0. + * + * This is an adapted to frontend version of + * RestoreArchivedFile function from transam/xlogarchive.c + */ +bool +RestoreArchivedWAL(const char *path, const char *xlogfname, off_t expectedSize, + const char *restoreCommand, const char *lastRestartPointFname) +{ + char xlogpath[MAXPGPATH]; + char xlogRestoreCmd[MAXPGPATH]; + char *dp; + char *endp; + const char *sp; + int rc; + struct stat stat_buf; + + snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname); + + /* + * Construct the command to be executed. + */ + dp = xlogRestoreCmd; + endp = xlogRestoreCmd + MAXPGPATH - 1; + *endp = '\0'; + + for (sp = restoreCommand; *sp; sp++) + { + if (*sp == '%') + { + switch (sp[1]) + { + case 'p': + /* %p: relative path of target file */ + sp++; + StrNCpy(dp, xlogpath, endp - dp); + make_native_path(dp); + dp += strlen(dp); + break; + case 'f': + /* %f: filename of desired file */ + sp++; + StrNCpy(dp, xlogfname, endp - dp); + dp += strlen(dp); + break; + case 'r': + /* %r: filename of last restartpoint */ + sp++; + StrNCpy(dp, lastRestartPointFname, endp - dp); + dp += strlen(dp); + break; + case '%': + /* convert %% to a single % */ + sp++; + if (dp < endp) + *dp++ = *sp; + break; + default: + /* otherwise treat the % as not special */ + if (dp < endp) + *dp++ = *sp; + break; + } + } + else + { + if (dp < endp) + *dp++ = *sp; + } + } + *dp = '\0'; + + /* + * Execute restore_command, which should copy + * the missing WAL file from archival storage. + */ + rc = system(xlogRestoreCmd); + + if (rc == 0) + { + /* + * Command apparently succeeded, but let's make sure the file is + * really there now and has the correct size. + */ + if (stat(xlogpath, &stat_buf) == 0) + { + if (expectedSize > 0 && stat_buf.st_size != expectedSize) + { + printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"), + xlogfname, (unsigned long) stat_buf.st_size, (unsigned long) expectedSize, + strerror(errno)); + } + else + return true; + } + else + { + /* Stat failed */ + printf(_("could not stat file \"%s\": %s"), + xlogpath, + strerror(errno)); + } + } + + return false; +} diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 9653106386..816a91fd78 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -19,11 +19,13 @@ #include "file_ops.h" #include "filemap.h" #include "logging.h" +#include "guc-file-fe.h" #include "access/timeline.h" #include "access/xlog_internal.h" #include "catalog/catversion.h" #include "catalog/pg_control.h" +#include "common/controldata_utils.h" #include "common/file_perm.h" #include "common/file_utils.h" #include "common/restricted_token.h" @@ -52,11 +54,13 @@ int WalSegSz; char *datadir_target = NULL; char *datadir_source = NULL; char *connstr_source = NULL; +char *restore_command = NULL; bool debug = false; bool showprogress = false; bool dry_run = false; bool do_sync = true; +bool restore_wals = false; /* Target history */ TimeLineHistoryEntry *targetHistory; @@ -75,6 +79,9 @@ usage(const char *progname) printf(_(" -N, --no-sync do not wait for changes to be written\n")); printf(_(" safely to disk\n")); printf(_(" -P, --progress write progress messages\n")); + printf(_(" -r, --use-postgresql-conf use restore_command in the postgresql.conf to\n")); + printf(_(" retreive WALs from archive\n")); + printf(_(" -R, --restore-command=COMMAND restore command\n")); printf(_(" --debug write a lot of debug messages\n")); printf(_(" -V, --version output version information, then exit\n")); printf(_(" -?, --help show this help, then exit\n")); @@ -94,9 +101,12 @@ main(int argc, char **argv) {"dry-run", no_argument, NULL, 'n'}, {"no-sync", no_argument, NULL, 'N'}, {"progress", no_argument, NULL, 'P'}, + {"use-postgresql-conf", no_argument, NULL, 'r'}, + {"restore-command", required_argument, NULL, 'R'}, {"debug", no_argument, NULL, 3}, {NULL, 0, NULL, 0} }; + char recfile_fullpath[MAXPGPATH]; int option_index; int c; XLogRecPtr divergerec; @@ -129,7 +139,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "DR:nNPr", long_options, &option_index)) != -1) { switch (c) { @@ -140,6 +150,10 @@ main(int argc, char **argv) case 'P': showprogress = true; break; + + case 'r': + restore_wals = true; + break; case 'n': dry_run = true; @@ -157,6 +171,10 @@ main(int argc, char **argv) datadir_target = pg_strdup(optarg); break; + case 'R': + restore_command = pg_strdup(optarg); + break; + case 1: /* --source-pgdata */ datadir_source = pg_strdup(optarg); break; @@ -223,6 +241,71 @@ main(int argc, char **argv) umask(pg_mode_mask); + if (restore_command != NULL) + { + pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command); + } + else if (restore_wals) + { + FILE *recovery_conf_file; + + /* + * Look for recovery.conf in the target data directory and + * try to get restore_command from there. + */ + snprintf(recfile_fullpath, sizeof(recfile_fullpath), "%s/%s", datadir_target, RECOVERY_COMMAND_FILE); + recovery_conf_file = fopen(recfile_fullpath, "r"); + + if (recovery_conf_file == NULL) + { + fprintf(stderr, _("%s: option -r/--use-postgresql-conf is specified, but postgreslq.conf is absent in the target directory\n"), + progname); + fprintf(stdout, _("You have to add postgresql.conf or pass restore_command with -R/--restore-command option.\n")); + exit(1); + } + else + { + ConfigVariable *item, + *head = NULL, + *tail = NULL; + bool config_is_parsed; + + /* + * We pass a fullpath to the recovery.conf as calling_file here, since + * parser will use its parent directory as base for all further includes + * if any exist. + */ + config_is_parsed = ParseConfigFile(RECOVERY_COMMAND_FILE, true, + recfile_fullpath, 0, 0, + PG_WARNING, &head, &tail); + fclose(recovery_conf_file); + + if (config_is_parsed) + { + for (item = head; item; item = item->next) + { + if (strcmp(item->name, "restore_command") == 0) + { + if (restore_command != NULL) + { + pfree(restore_command); + restore_command = NULL; + } + restore_command = pstrdup(item->value); + pg_log(PG_DEBUG, "using restore_command=\'%s\' from recovery.conf.\n", restore_command); + } + } + + if (restore_command == NULL) + pg_fatal("could not find restore_command in recovery.conf file %s\n", recfile_fullpath); + } + else + pg_fatal("could not parse recovery.conf file %s\n", recfile_fullpath); + + FreeConfigVariables(head); + } + } + /* Connect to remote server */ if (connstr_source) libpqConnect(connstr_source); @@ -294,9 +377,9 @@ main(int argc, char **argv) exit(0); } - findLastCheckpoint(datadir_target, divergerec, + findLastCheckpoint(datadir_target, &ControlFile_target, divergerec, lastcommontliIndex, - &chkptrec, &chkpttli, &chkptredo); + &chkptrec, &chkpttli, &chkptredo, restore_command); printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"), (uint32) (chkptrec >> 32), (uint32) chkptrec, chkpttli); @@ -319,7 +402,7 @@ main(int argc, char **argv) */ pg_log(PG_PROGRESS, "reading WAL in target\n"); extractPageMap(datadir_target, chkptrec, lastcommontliIndex, - ControlFile_target.checkPoint); + &ControlFile_target, restore_command); filemap_finalize(); if (showprogress) diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 3f4ba7a267..ba9d13e9bd 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -14,6 +14,7 @@ #include "datapagemap.h" #include "access/timeline.h" +#include "catalog/pg_control.h" #include "storage/block.h" #include "storage/relfilenode.h" @@ -32,11 +33,10 @@ extern int targetNentries; /* in parsexlog.c */ extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, - int tliIndex, XLogRecPtr endpoint); -extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr, - int tliIndex, - XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo); + int tliIndex, ControlFileData *targetCF, const char *restoreCommand); +extern void findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr searchptr, + int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, + XLogRecPtr *lastchkptredo, const char *restoreCommand); extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex); diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl index 53dbf45be2..22777bff82 100644 --- a/src/bin/pg_rewind/t/001_basic.pl +++ b/src/bin/pg_rewind/t/001_basic.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 10; +use Test::More tests => 15; use RewindTest; @@ -103,5 +103,6 @@ in master, before promotion # Run the test in both modes run_test('local'); run_test('remote'); +run_test('archive'); exit(0); diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl index 2c9e427831..10c835efc1 100644 --- a/src/bin/pg_rewind/t/002_databases.pl +++ b/src/bin/pg_rewind/t/002_databases.pl @@ -1,7 +1,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 6; +use Test::More tests => 9; use RewindTest; @@ -59,5 +59,6 @@ template1 # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); exit(0); diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl index 496f38c457..f94ce329f4 100644 --- a/src/bin/pg_rewind/t/003_extrafiles.pl +++ b/src/bin/pg_rewind/t/003_extrafiles.pl @@ -3,7 +3,7 @@ use strict; use warnings; use TestLib; -use Test::More tests => 4; +use Test::More tests => 6; use File::Find; @@ -87,5 +87,6 @@ sub run_test # Run the test in both modes. run_test('local'); run_test('remote'); +run_test('archive'); exit(0); diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 2921d193a1..3da0fdc956 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -73,6 +73,7 @@ my $frontend_extraincludes = { }; my $frontend_extrasource = { 'psql' => ['src/bin/psql/psqlscanslash.l'], + 'pg_rewind' => ['src/bin/pg_rewind/guc-file-fe.l'], 'pgbench' => [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ] }; -- 2.17.1 --------------2FDD67D29E17444B4089892D--