agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v2 5/5] Allow pg_rewind to use a standby server as the source system. 21+ messages / 2 participants [nested] [flat]
* [PATCH v2 5/5] Allow pg_rewind to use a standby server as the source system. @ 2020-08-19 12:34 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Heikki Linnakangas @ 2020-08-19 12:34 UTC (permalink / raw) Using a hot standby server as the source has not been possible, because pg_rewind creates a temporary table in the source system, to hold the list of file ranges that need to be fetched. Refactor it to queue up the file fetch requests in pg_rewind's memory, so that the temporary table is no longer needed. Also update the logic to compute 'minRecoveryPoint' correctly, when the source is a standby server. --- src/bin/pg_rewind/libpq_fetch.c | 279 ++++++++++++++++++++++---------- src/bin/pg_rewind/pg_rewind.c | 67 ++++++-- 2 files changed, 252 insertions(+), 94 deletions(-) diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 52c4e147e10..f61a4242999 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -15,39 +15,61 @@ #include "fetch.h" #include "file_ops.h" #include "filemap.h" +#include "lib/stringinfo.h" #include "pg_rewind.h" #include "port/pg_bswap.h" /* - * Files are fetched max CHUNKSIZE bytes at a time. - * - * (This only applies to files that are copied in whole, or for truncated - * files where we copy the tail. Relation files, where we know the individual - * blocks that need to be fetched, are fetched in BLCKSZ chunks.) + * Files are fetched max CHUNK_SIZE bytes at a time, and with a + * maximum of MAX_CHUNKS_PER_QUERY chunks in a single query. */ -#define CHUNKSIZE 1000000 +#define CHUNK_SIZE (1024 * 1024) +#define MAX_CHUNKS_PER_QUERY 1000 + +/* represents the request to fetch a piece of a file from the source */ +typedef struct +{ + const char *path; /* path relative to data directory root */ + uint64 offset; + uint32 length; +} fetch_range_request; typedef struct { rewind_source common; /* common interface functions */ PGconn *conn; + + /* + * Queue of chunks that have been requested with the queue_fetch_range() + * function, but have not been fetched from the remote server yet. + */ + int num_requests; + fetch_range_request request_queue[MAX_CHUNKS_PER_QUERY]; + + /* temporary space for process_queued_fetch_requests() */ + StringInfoData paths; + StringInfoData offsets; + StringInfoData lengths; } libpq_source; static void init_libpq_conn(PGconn *conn); static char *run_simple_query(PGconn *conn, const char *sql); static void run_simple_command(PGconn *conn, const char *sql); +static void appendArrayEscapedString(StringInfo buf, const char *str); + +static void process_queued_fetch_requests(libpq_source *src); /* public interface functions */ static void libpq_traverse_files(rewind_source *source, process_file_callback_t callback); -static char *libpq_fetch_file(rewind_source *source, const char *path, - size_t *filesize); static void libpq_queue_fetch_range(rewind_source *source, const char *path, uint64 off, size_t len); static void libpq_finish_fetch(rewind_source *source); -static void libpq_destroy(rewind_source *source); +static char *libpq_fetch_file(rewind_source *source, const char *path, + size_t *filesize); static XLogRecPtr libpq_get_current_wal_insert_lsn(rewind_source *source); +static void libpq_destroy(rewind_source *source); /* * Create a new libpq source. @@ -73,6 +95,10 @@ init_libpq_source(PGconn *conn) src->conn = conn; + initStringInfo(&src->paths); + initStringInfo(&src->offsets); + initStringInfo(&src->lengths); + return &src->common; } @@ -90,7 +116,10 @@ init_libpq_conn(PGconn *conn) run_simple_command(conn, "SET lock_timeout = 0"); run_simple_command(conn, "SET idle_in_transaction_session_timeout = 0"); - /* we don't intend do any updates. Put the connection in read-only mode to keep us honest */ + /* + * We don't intend do any updates. Put the connection in read-only mode + * to keep us honest. + */ run_simple_command(conn, "SET default_transaction_read_only = off"); /* secure search_path */ @@ -100,17 +129,6 @@ init_libpq_conn(PGconn *conn) PQresultErrorMessage(res)); PQclear(res); - /* - * Check that the server is not in hot standby mode. There is no - * fundamental reason that couldn't be made to work, but it doesn't - * currently because we use a temporary table. Better to check for it - * explicitly than error out, for a better error message. - */ - str = run_simple_query(conn, "SELECT pg_is_in_recovery()"); - if (strcmp(str, "f") != 0) - pg_fatal("source server must not be in recovery mode"); - pg_free(str); - /* * Also check that full_page_writes is enabled. We can get torn pages if * a page is modified while we read it with pg_read_binary_file(), and we @@ -121,15 +139,15 @@ init_libpq_conn(PGconn *conn) pg_fatal("full_page_writes must be enabled in the source server"); pg_free(str); - /* - * First create a temporary table, and COPY to load it with the list of - * blocks that we need to fetch. - */ - run_simple_command(conn, "CREATE TEMPORARY TABLE fetchchunks(path text, begin int8, len int4)"); + /* Prepare a statement we'll use to fetch files */ + res = PQprepare(conn, "fetch_chunks_stmt", + "SELECT path, begin,\n" + " pg_read_binary_file(path, begin, len, true) AS chunk\n" + "FROM unnest ($1::text[], $2::int8[], $3::int4[]) as x(path, begin, len)", + 3, NULL); - res = PQexec(conn, "COPY fetchchunks FROM STDIN"); - if (PQresultStatus(res) != PGRES_COPY_IN) - pg_fatal("could not send file list: %s", + if (PQresultStatus(res) != PGRES_COMMAND_OK) + pg_fatal("could not prepare statement to fetch file contents: %s", PQresultErrorMessage(res)); PQclear(res); } @@ -297,91 +315,143 @@ libpq_queue_fetch_range(rewind_source *source, const char *path, uint64 off, size_t len) { libpq_source *src = (libpq_source *) source; - uint64 begin = off; - uint64 end = off + len; /* - * Write the file range to a temporary table in the server. + * Does this request happen to be a continuation of the previous chunk? + * If so, merge it with the previous one. * - * The range is sent to the server as a COPY formatted line, to be inserted - * into the 'fetchchunks' temporary table. The libpq_finish_fetch() uses - * the temporary table to actually fetch the data. + * XXX: We use pointer equality to compare the path. That's good enough + * for our purposes; the caller always passes the same pointer for the + * same filename. If it didn't, we would fail to merge requests, but it + * wouldn't affect correctness. */ - - /* Split the range into CHUNKSIZE chunks */ - while (end - begin > 0) + if (src->num_requests > 0) { - char linebuf[MAXPGPATH + 23]; - unsigned int len; + fetch_range_request *prev = &src->request_queue[src->num_requests - 1]; - /* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */ - if (end - begin > CHUNKSIZE) - len = CHUNKSIZE; - else - len = (unsigned int) (end - begin); + if (prev->offset + prev->length == off && + prev->length < CHUNK_SIZE && + prev->path == path) + { + /* + * Extend the previous request to cover as much of this new request + * as possible, without exceeding CHUNK_SIZE. + */ + int32 thislen; + + thislen = Min(len, CHUNK_SIZE - prev->length); + src->request_queue[src->num_requests - 1].length += thislen; - begin += len; + off += thislen; + len -= thislen; - snprintf(linebuf, sizeof(linebuf), "%s\t" UINT64_FORMAT "\t%u\n", path, begin, len); + /* + * Fall through to create new requests for any remaining 'len' that + * didn't fit in the previous chunk. + */ + } + } + + /* Divide the request into pieces of CHUNK_SIZE bytes each */ + while (len > 0) + { + int32 thislen; + + /* if the queue is full, perform all the work queued up so far */ + if (src->num_requests == MAX_CHUNKS_PER_QUERY) + process_queued_fetch_requests(src); + + thislen = Min(len, CHUNK_SIZE); + src->request_queue[src->num_requests].path = path; + src->request_queue[src->num_requests].offset = off; + src->request_queue[src->num_requests].length = thislen; + src->num_requests++; - if (PQputCopyData(src->conn, linebuf, strlen(linebuf)) != 1) - pg_fatal("could not send COPY data: %s", - PQerrorMessage(src->conn)); + off += thislen; + len -= thislen; } } - -/*---- - * Runs a query, which returns pieces of files from the remote source data - * directory, and overwrites the corresponding parts of target files with - * the received parts. The result set is expected to be of format: - * - * path text -- path in the data directory, e.g "base/1/123" - * begin int8 -- offset within the file - * chunk bytea -- file content - *---- +/* + * Fetche all the queued chunks and writes them to the target data directory. */ +static void +libpq_finish_fetch(rewind_source *source) +{ + process_queued_fetch_requests((libpq_source *) source); +} + /* * Receive all the queued chunks and write them to the target data directory. */ static void -libpq_finish_fetch(rewind_source *source) +process_queued_fetch_requests(libpq_source *src) { - libpq_source *src = (libpq_source *) source; + const char *params[3]; PGresult *res; - const char *sql; + int chunkno; - if (PQputCopyEnd(src->conn, NULL) != 1) - pg_fatal("could not send end-of-COPY: %s", - PQerrorMessage(src->conn)); + if (src->num_requests == 0) + return; - while ((res = PQgetResult(src->conn)) != NULL) + pg_log_debug("getting %d file chunks", src->num_requests); + + /* + * The prepared statement, 'fetch_chunks_stmt', takes three arrays + * with the same length as parameters: paths, offsets and lengths. + * Construct the string representations of the parameter arrays. + */ + resetStringInfo(&src->paths); + resetStringInfo(&src->offsets); + resetStringInfo(&src->lengths); + + appendStringInfoChar(&src->paths, '{'); + appendStringInfoChar(&src->offsets, '{'); + appendStringInfoChar(&src->lengths, '{'); + for (int i = 0; i < src->num_requests; i++) { - if (PQresultStatus(res) != PGRES_COMMAND_OK) - pg_fatal("unexpected result while sending file list: %s", - PQresultErrorMessage(res)); - PQclear(res); + fetch_range_request *rq = &src->request_queue[i]; + + if (i > 0) + { + appendStringInfoChar(&src->paths, ','); + appendStringInfoChar(&src->offsets, ','); + appendStringInfoChar(&src->lengths, ','); + } + + appendArrayEscapedString(&src->paths, rq->path); + appendStringInfo(&src->offsets, INT64_FORMAT, rq->offset); + appendStringInfo(&src->lengths, "%d", rq->length); } + appendStringInfoChar(&src->paths, '}'); + appendStringInfoChar(&src->offsets, '}'); + appendStringInfoChar(&src->lengths, '}'); /* - * We've now copied the list of file ranges that we need to fetch to the - * temporary table. Now, actually fetch all of those ranges. + * Execute the prepared statement. */ - sql = - "SELECT path, begin,\n" - " pg_read_binary_file(path, begin, len, true) AS chunk\n" - "FROM fetchchunks\n"; + params[0] = src->paths.data; + params[1] = src->offsets.data; + params[2] = src->lengths.data; - if (PQsendQueryParams(src->conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1) + if (PQsendQueryPrepared(src->conn, "fetch_chunks_stmt", 3, params, NULL, NULL, 1) != 1) pg_fatal("could not send query: %s", PQerrorMessage(src->conn)); - pg_log_debug("getting file chunks"); - if (PQsetSingleRowMode(src->conn) != 1) pg_fatal("could not set libpq connection to single row mode"); + /*---- + * The result set is of format: + * + * path text -- path in the data directory, e.g "base/1/123" + * begin int8 -- offset within the file + * chunk bytea -- file content + *---- + */ + chunkno = 0; while ((res = PQgetResult(src->conn)) != NULL) { + fetch_range_request *rq = &src->request_queue[chunkno]; char *filename; int filenamelen; int64 chunkoff; @@ -402,6 +472,9 @@ libpq_finish_fetch(rewind_source *source) PQresultErrorMessage(res)); } + if (chunkno > src->num_requests) + pg_fatal("received more data chunks than requested"); + /* sanity check the result set */ if (PQnfields(res) != 3 || PQntuples(res) != 1) pg_fatal("unexpected result set size while fetching remote files"); @@ -446,9 +519,7 @@ libpq_finish_fetch(rewind_source *source) * If a file has been deleted on the source, remove it on the target * as well. Note that multiple unlink() calls may happen on the same * file if multiple data chunks are associated with it, hence ignore - * unconditionally anything missing. If this file is not a relation - * data file, then it has been already truncated when creating the - * file chunk list at the previous execution of the filemap. + * unconditionally anything missing. */ if (PQgetisnull(res, 0, 2)) { @@ -463,14 +534,54 @@ libpq_finish_fetch(rewind_source *source) pg_log_debug("received chunk for file \"%s\", offset %lld, size %d", filename, (long long int) chunkoff, chunksize); - open_target_file(filename, false); + if (strcmp(filename, rq->path) != 0) + { + pg_fatal("received data for file \"%s\", when requested for \"%s\"", + filename, rq->path); + } + if (chunkoff != rq->offset) + pg_fatal("received data at offset " UINT64_FORMAT" of file \"%s\", when requested for offset " UINT64_FORMAT, + chunkoff, rq->path, rq->offset); + if (chunksize > rq->length) + { + pg_fatal("received more than requested for file \"%s\"", + rq->path); + /* receiving less is OK, though */ + } + open_target_file(filename, false); write_target_range(chunk, chunkoff, chunksize); pg_free(filename); PQclear(res); + chunkno++; + } + if (chunkno != src->num_requests) + pg_fatal("unexpected number of data chunks received"); + + src->num_requests = 0; +} + +/* + * Escape a string to be used as element in a text array constant + */ +static void +appendArrayEscapedString(StringInfo buf, const char *str) +{ + appendStringInfoCharMacro(buf, '\"'); + while (*str) + { + char ch = *str; + + if (ch == '"' || ch == '\\') + appendStringInfoCharMacro(buf, '\\'); + + appendStringInfoCharMacro(buf, ch); + + str++; } + appendStringInfoCharMacro(buf, '\"'); } /* diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 9e04a085226..e9a3481be58 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -45,6 +45,7 @@ static void disconnect_atexit(void); static ControlFileData ControlFile_target; static ControlFileData ControlFile_source; +static ControlFileData ControlFile_source_after; const char *progname; int WalSegSz; @@ -446,30 +447,76 @@ main(int argc, char **argv) progress_report(true); + /* + * Fetch the control file from the source last. This ensures that the + * minRecoveryPoint is up-to-date. + */ + buffer = source->fetch_file(source, "global/pg_control", &size); + digestControlFile(&ControlFile_source_after, buffer, size); + pg_free(buffer); + + /* + * Sanity check: If the source is a local system, the control file should + * not have changed since we started. + * + * XXX: We assume it hasn't been modified, but actually, what could go + * wrong? The logic handles a libpq source that's modified concurrently, + * why not a local datadir? + */ + if (datadir_source && + memcmp(&ControlFile_source, &ControlFile_source_after, + sizeof(ControlFileData)) != 0) + { + pg_fatal("source system was modified while pg_rewind was running"); + } + if (showprogress) pg_log_info("creating backup label and updating control file"); createBackupLabel(chkptredo, chkpttli, chkptrec); /* * Update control file of target. Make it ready to perform archive - * recovery when restarting. + * recovery when restarting, starting from the last common checkpoint. * - * minRecoveryPoint is set to the current WAL insert location in the - * source server. Like in an online backup, it's important that we recover - * all the WAL that was generated while we copied the files over. + * Like in an online backup, it's important that we replay all the WAL + * that was generated while we copied the files over. To enforce that, + * set 'minRecoveryPoint' in the control file. */ - memcpy(&ControlFile_new, &ControlFile_source, sizeof(ControlFileData)); - if (connstr_source) { - endrec = source->get_current_wal_insert_lsn(source); - endtli = ControlFile_source.checkPointCopy.ThisTimeLineID; + if (ControlFile_source_after.state == DB_IN_ARCHIVE_RECOVERY) + { + /* + * Source is a standby server. We must replay to its + * minRecoveryPoint. + */ + endrec = ControlFile_source_after.minRecoveryPoint; + endtli = ControlFile_source_after.minRecoveryPointTLI; + } + else + { + /* + * Source is a production, non-standby, server. We must recover up + * to the last WAL insert location. + */ + if (ControlFile_source_after.state != DB_IN_PRODUCTION) + pg_fatal("source system was in unexpected state at end of rewind"); + + endrec = source->get_current_wal_insert_lsn(source); + endtli = ControlFile_source_after.checkPointCopy.ThisTimeLineID; + } } else { - endrec = ControlFile_source.checkPoint; - endtli = ControlFile_source.checkPointCopy.ThisTimeLineID; + /* + * Source is a local data directory. It should've shut down cleanly, + * and we must to the latest shutdown checkpoint. + */ + endrec = ControlFile_source_after.checkPoint; + endtli = ControlFile_source_after.checkPointCopy.ThisTimeLineID; } + + memcpy(&ControlFile_new, &ControlFile_source_after, sizeof(ControlFileData)); ControlFile_new.minRecoveryPoint = endrec; ControlFile_new.minRecoveryPointTLI = endtli; ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY; -- 2.20.1 --------------BF34D0120055DC3839060F92-- ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 2/3] cirrus/windows: add compiler_warnings_script @ 2022-02-13 00:59 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-02-13 00:59 UTC (permalink / raw) ci-os-only: windows https://cirrus-ci.com/task/6316260295180288 --- .cirrus.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index eda8ac9596c..ea87c890cc3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -397,7 +397,7 @@ task: - perl src/tools/msvc/mkvcbuild.pl build_script: - vcvarsall x64 - - msbuild %MSBFLAGS% pgsql.sln + - msbuild %MSBFLAGS% pgsql.sln >build.log tempinstall_script: # Installation on windows currently only completely works from src/tools/msvc - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install @@ -440,6 +440,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + always: + compiler_warnings_script: | + findstr "warning " build.log && exit /b 1 + exit /b 0 + on_failure: <<: *on_failure crashlog_artifacts: -- 2.17.1 --1XWsVB21DFCvn2e8 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-cirrus-upload-changed-html-docs-as-artifacts.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 2/8] cirrus/windows: add compiler_warnings_script @ 2022-02-13 00:59 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-02-13 00:59 UTC (permalink / raw) ci-os-only: windows https://cirrus-ci.com/task/6316260295180288 --- .cirrus.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index eda8ac9596c..2d023a31d2d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -397,7 +397,7 @@ task: - perl src/tools/msvc/mkvcbuild.pl build_script: - vcvarsall x64 - - msbuild %MSBFLAGS% pgsql.sln + - msbuild %MSBFLAGS% pgsql.sln |tee build.log tempinstall_script: # Installation on windows currently only completely works from src/tools/msvc - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install @@ -440,6 +440,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + always: + compiler_warnings_script: | + findstr "warning " build.log && exit /b 1 + exit /b 0 + on_failure: <<: *on_failure crashlog_artifacts: -- 2.17.1 --4e5ZDkbgLEOfWmLx Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-cirrus-upload-changed-html-docs-as-artifacts.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 3/7] cirrus/windows: add compiler_warnings_script @ 2022-02-20 21:01 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-02-20 21:01 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 9 ++++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index 5179353dc9d..a3af4a0a808 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -366,7 +366,8 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -447,6 +448,12 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + # Success if the file doesn't exist or is empty, else fail + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --wLAMOaPNJ0fu1fTG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-cirrus-code-coverage.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/23] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 14 +++++++++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index 81eb8a9996d..2be62791448 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -370,7 +370,14 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: >- + "-consoleLoggerParameters:Summary;ForceNoAlign" + -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log + -m + -nologo + -p:TrackFileAccess=false + -verbosity:minimal # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -450,6 +457,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --tKkaNMvYmhQvRCRK Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-vcregress-test-modules-contrib-with-NO_INSTAL.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 4/8] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 10 +++++++++- src/tools/ci/windows-compiler-warnings | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index 9f2282471a9..99ac09dc679 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -451,12 +451,20 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build/meson-logs/build.txt + REM Since pipes lose exit status of the preceding command, rerun compilation, + REM without the pipe exiting now if it fails, rather than trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings build/meson-logs/build.txt build/meson-logs/build-warnings.txt + on_failure: <<: *on_failure_meson crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..685aa995d5c --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,24 @@ +#! /bin/sh +# Success if the given exists and is empty, else fail +# This is a separate file to avoid dealing with windows shell quoting and escaping, +# or giving the impression that one-liner scripts will work trivially. +set -e + +infile=$1 +outfile=$2 + +# Looks like: +# [19:39:27.130] c:\cirrus\src\backend\backup\basebackup_zstd.c(80) : warning C4715: 'bbsink_zstd_new': not all control paths return a value +# should include linker warnings? +grep ": warning " "$infile" > $outfile || + [ $? -eq 1 ] + +if [ -s "$outfile" ] +then + # Display the file's content, then exit indicating failure + cat "$outfile" + exit 1 +else + # Success + exit 0 +fi -- 2.25.1 --rPFbv4B2w2tcwGo5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0005-cirrus-build-docs-as-a-separate-task.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/10] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index f31923333ef..6ce4f393e29 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -547,12 +547,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.25.1 --sMkrXc3gAYLRVOjR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-macos-switch-to-macos_instance-M1.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/19] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 8 +++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index f23d6cae552..bcb8d53db78 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -370,7 +370,8 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -450,6 +451,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --Sw7tCqrGA+HQ0/zt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-vcregress-test-modules-contrib-with-NO_INSTAL.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/11] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 10 +++++++++- src/tools/ci/windows-compiler-warnings | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index 9f2282471a9..7596b2058c6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -451,12 +451,20 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build/meson-logs/build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings build/meson-logs/build.txt build/meson-logs/build-warnings.txt + on_failure: <<: *on_failure_meson crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..685aa995d5c --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,24 @@ +#! /bin/sh +# Success if the given exists and is empty, else fail +# This is a separate file to avoid dealing with windows shell quoting and escaping, +# or giving the impression that one-liner scripts will work trivially. +set -e + +infile=$1 +outfile=$2 + +# Looks like: +# [19:39:27.130] c:\cirrus\src\backend\backup\basebackup_zstd.c(80) : warning C4715: 'bbsink_zstd_new': not all control paths return a value +# should include linker warnings? +grep ": warning " "$infile" > $outfile || + [ $? -eq 1 ] + +if [ -s "$outfile" ] +then + # Display the file's content, then exit indicating failure + cat "$outfile" + exit 1 +else + # Success + exit 0 +fi -- 2.25.1 --I4VOKWutKNZEOIPu Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-build-docs-as-a-separate-task.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/6] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 [email protected] [email protected] [email protected] [email protected] [email protected] ci-os-only: windows --- .cirrus.tasks.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 33646faeadf..5a2b64f64c2 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -566,12 +566,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.42.0 --Rvs4meNCoh6yKfx/ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-WIP-cirrus-windows-ccache.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/6] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 [email protected] [email protected] [email protected] [email protected] [email protected] ci-os-only: windows --- .cirrus.tasks.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 33646faeadf..5a2b64f64c2 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -566,12 +566,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.42.0 --AZUs1t8VY6GH6Ct3 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-WIP-cirrus-windows-ccache.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/7] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 [email protected] [email protected] [email protected] [email protected] [email protected] ci-os-only: windows --- .cirrus.tasks.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 33646faeadf..5a2b64f64c2 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -566,12 +566,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.42.0 --jviKHM0HgsS9J3Sv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-windows-ccache.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/6] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.tasks.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index e4e1bcfeb99..2d397448851 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -561,12 +561,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.42.0 --r+4ArEqyVZuXSt++ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-002_pg_upgrade-exercise-link-and-clone.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/8] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index d3f88821a85..13213ffd304 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -589,28 +589,37 @@ task: echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts echo 127.0.0.3 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts type c:\Windows\System32\Drivers\etc\hosts # Use /DEBUG:FASTLINK to avoid high memory usage during linking configure_script: | vcvarsall x64 meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: path: "crashlog-*.txt" type: text/plain task: << : *WINDOWS_ENVIRONMENT_BASE name: Windows - Server 2019, MinGW64 - Meson -- 2.34.1 --NvorufYXl2Cnpa+A Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-freebsd-run-with-more-CPUs-RAM-and-do-not-rep.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/9] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 1204824d2eb..eefc5c21fe6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -566,12 +566,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.25.1 --61jdw2sOBCFtR2d/ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-freebsd-run-with-more-CPUs-RAM-and-do-not-rep.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/7] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 69837bcd5ad..fd461048372 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -564,12 +564,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.25.1 --O98KdSgI27dgYlM5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-macos-update-to-macos-ventura.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/21] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 8 +++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index f23d6cae552..bcb8d53db78 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -370,7 +370,8 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -450,6 +451,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --STPqjqpCrtky8aYs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-vcregress-test-modules-contrib-with-NO_INSTAL.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/6] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) The goal is to fail due to warnings only after running tests. (At least historically, it's too slow to run a separate windows VM to compile with -Werror.) https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 [email protected] [email protected] [email protected] [email protected] [email protected] ci-os-only: windows --- .cirrus.tasks.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 1ce6c443a8c..5dd37e07aae 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -576,12 +576,21 @@ task: build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: -- 2.42.0 --ndI8Dx/wEX5btThR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-windows-ccache.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 1/8] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also easy to write something that doesn't work in posix sh, since windows shell is interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 https://cirrus-ci.com/task/6241060062494720 https://cirrus-ci.com/task/6496366607204352 ci-os-only: windows --- .cirrus.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 505c50f3285..60c0efc2e63 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -563,28 +563,37 @@ task: setup_additional_packages_script: | REM choco install -y --no-progress ... # Use /DEBUG:FASTLINK to avoid high memory usage during linking configure_script: | vcvarsall x64 meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build build_script: | vcvarsall x64 - ninja -C build + ninja -C build |tee build.txt + REM Since pipes lose the exit status of the preceding command, rerun the compilation + REM without the pipe, exiting now if it fails, to avoid trying to run checks + ninja -C build > nul check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% + # This should be last, so check_world is run even if there are warnings + always: + compiler_warnings_script: + # this avoids using metachars which would be interpretted by the windows shell + - sh -c 'if grep ": warning " build.txt; then exit 1; fi; exit 0' + on_failure: <<: *on_failure_meson crashlog_artifacts: path: "crashlog-*.txt" type: text/plain task: << : *WINDOWS_ENVIRONMENT_BASE name: Windows - Server 2019, MinGW64 - Meson -- 2.34.1 --Wo0n/IWMQc9EwEbt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-freebsd-run-with-more-CPUs-RAM-and-do-not-rep.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/21] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 8 +++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index f23d6cae552..bcb8d53db78 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -370,7 +370,8 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -450,6 +451,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --mln0rGgUGuXEqmuI Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-vcregress-test-modules-contrib-with-NO_INSTAL.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* [PATCH 01/25] cirrus/windows: add compiler_warnings_script @ 2022-05-26 02:53 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2022-05-26 02:53 UTC (permalink / raw) I'm not sure how to write this test in windows shell; it's also not easy to write it in posix sh, since windows shell is somehow interpretting && and ||... https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de See also: 8a1ce5e54f6d144e4f8e19af7c767b026ee0c956 ci-os-only: windows https://cirrus-ci.com/task/6183879907213312 https://cirrus-ci.com/task/4876271443247104 --- .cirrus.yml | 8 +++++++- src/tools/ci/windows-compiler-warnings | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 src/tools/ci/windows-compiler-warnings diff --git a/.cirrus.yml b/.cirrus.yml index 81eb8a9996d..da16344341b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -370,7 +370,8 @@ task: # ForceNoAlign prevents msbuild from introducing line-breaks for long lines # disable file tracker, we're never going to rebuild, and it slows down the # build - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo + # -fileLoggerParameters1: write warnings to msbuild.warn.log. + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log # If tests hang forever, cirrus eventually times out. In that case log # output etc is not uploaded, making the problem hard to debug. Of course @@ -450,6 +451,11 @@ task: cd src/tools/msvc %T_C% perl vcregress.pl ecpgcheck + # These should be last, so all the important checks are always run + always: + compiler_warnings_script: + - sh src\tools\ci\windows-compiler-warnings msbuild.warn.log + on_failure: <<: *on_failure crashlog_artifacts: diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings new file mode 100755 index 00000000000..d6f9a1fc569 --- /dev/null +++ b/src/tools/ci/windows-compiler-warnings @@ -0,0 +1,16 @@ +#! /bin/sh +# Success if the given file doesn't exist or is empty, else fail +# This is a separate file only to avoid dealing with windows shell quoting and escaping. +set -e + +fn=$1 + +if [ -s "$fn" ] +then + # Display the file's content, then exit indicating failure + cat "$fn" + exit 1 +else + # Success + exit 0 +fi -- 2.17.1 --IS0zKkzwUGydFO0o Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-cirrus-vcregress-test-modules-contrib-with-NO_INSTAL.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
end of thread, other threads:[~2022-05-26 02:53 UTC | newest] Thread overview: 21+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-08-19 12:34 [PATCH v2 5/5] Allow pg_rewind to use a standby server as the source system. Heikki Linnakangas <[email protected]> 2022-02-13 00:59 [PATCH 2/8] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-02-13 00:59 [PATCH 2/3] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-02-20 21:01 [PATCH 3/7] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/8] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/23] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/11] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/6] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/7] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/6] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/6] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 4/8] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/25] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/10] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/19] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/9] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/21] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/6] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/8] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 1/7] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]> 2022-05-26 02:53 [PATCH 01/21] cirrus/windows: add compiler_warnings_script Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox