agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v4 1/3] Implement REINDEX of partitioned tables/indexes 7+ messages / 1 participants [nested] [flat]
* [PATCH v4 1/3] Implement REINDEX of partitioned tables/indexes @ 2020-06-07 20:59 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2020-06-07 20:59 UTC (permalink / raw) --- doc/src/sgml/ref/reindex.sgml | 5 - src/backend/catalog/index.c | 7 +- src/backend/commands/indexcmds.c | 119 +++++++++++++-------- src/backend/tcop/utility.c | 6 +- src/include/commands/defrem.h | 6 +- src/test/regress/expected/create_index.out | 18 ++-- src/test/regress/sql/create_index.sql | 13 ++- 7 files changed, 106 insertions(+), 68 deletions(-) diff --git a/doc/src/sgml/ref/reindex.sgml b/doc/src/sgml/ref/reindex.sgml index aac5d5be23..e2e32b3ba0 100644 --- a/doc/src/sgml/ref/reindex.sgml +++ b/doc/src/sgml/ref/reindex.sgml @@ -258,11 +258,6 @@ REINDEX [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] { IN can always reindex anything. </para> - <para> - Reindexing partitioned tables or partitioned indexes is not supported. - Each individual partition can be reindexed separately instead. - </para> - <refsect2 id="sql-reindex-concurrently" xreflabel="Rebuilding Indexes Concurrently"> <title>Rebuilding Indexes Concurrently</title> diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 1be27eec52..7abd7f5cfc 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -3660,12 +3660,7 @@ reindex_relation(Oid relid, int flags, int options) */ rel = table_open(relid, ShareLock); - /* - * This may be useful when implemented someday; but that day is not today. - * For now, avoid erroring out when called in a multi-table context - * (REINDEX SCHEMA) and happen to come across a partitioned table. The - * partitions may be reindexed on their own anyway. - */ + /* Avoid erroring out */ if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) { ereport(WARNING, diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 2baca12c5f..df3e567acb 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -88,7 +88,8 @@ static List *ChooseIndexColumnNames(List *indexElems); static void RangeVarCallbackForReindexIndex(const RangeVar *relation, Oid relId, Oid oldRelId, void *arg); static bool ReindexRelationConcurrently(Oid relationOid, int options); -static void ReindexPartitionedIndex(Relation parentIdx); +static void ReindexPartitionedRel(Oid reloid, int options, bool concurrent, + bool isTopLevel); static void update_relispartition(Oid relationId, bool newval); static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts); @@ -2420,11 +2421,10 @@ ChooseIndexColumnNames(List *indexElems) * Recreate a specific index. */ void -ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) +ReindexIndex(RangeVar *indexRelation, int options, bool concurrent, bool isTopLevel) { struct ReindexIndexCallbackState state; Oid indOid; - Relation irel; char persistence; /* @@ -2445,22 +2445,10 @@ ReindexIndex(RangeVar *indexRelation, int options, bool concurrent) RangeVarCallbackForReindexIndex, &state); - /* - * Obtain the current persistence of the existing index. We already hold - * lock on the index. - */ - irel = index_open(indOid, NoLock); - - if (irel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) - { - ReindexPartitionedIndex(irel); - return; - } - - persistence = irel->rd_rel->relpersistence; - index_close(irel, NoLock); - - if (concurrent && persistence != RELPERSISTENCE_TEMP) + persistence = get_rel_persistence(indOid); + if (get_rel_relkind(indOid) == RELKIND_PARTITIONED_INDEX) + ReindexPartitionedRel(indOid, options, concurrent, isTopLevel); + else if (concurrent && persistence != RELPERSISTENCE_TEMP) ReindexRelationConcurrently(indOid, options); else reindex_index(indOid, false, persistence, @@ -2542,7 +2530,7 @@ RangeVarCallbackForReindexIndex(const RangeVar *relation, * Recreate all indexes of a table (and of its toast table, if any) */ Oid -ReindexTable(RangeVar *relation, int options, bool concurrent) +ReindexTable(RangeVar *relation, int options, bool concurrent, bool isTopLevel) { Oid heapOid; bool result; @@ -2560,7 +2548,9 @@ ReindexTable(RangeVar *relation, int options, bool concurrent) 0, RangeVarCallbackOwnsTable, NULL); - if (concurrent && get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) + if (get_rel_relkind(heapOid) == RELKIND_PARTITIONED_TABLE) + ReindexPartitionedRel(heapOid, options, concurrent, isTopLevel); + else if (concurrent && get_rel_persistence(heapOid) != RELPERSISTENCE_TEMP) { result = ReindexRelationConcurrently(heapOid, options); @@ -2688,11 +2678,8 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * Only regular tables and matviews can have indexes, so ignore any * other kind of relation. * - * It is tempting to also consider partitioned tables here, but that - * has the problem that if the children are in the same schema, they - * would be processed twice. Maybe we could have a separate list of - * partitioned tables, and expand that afterwards into relids, - * ignoring any duplicates. + * Partitioned tables/indexes are skipped but matching leaf + * partitions are processed. */ if (classtuple->relkind != RELKIND_RELATION && classtuple->relkind != RELKIND_MATVIEW) @@ -2805,8 +2792,7 @@ ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, * view. For tables and materialized views, all its indexes will be rebuilt, * excluding invalid indexes and any indexes used in exclusion constraints, * but including its associated toast table indexes. For indexes, the index - * itself will be rebuilt. If 'relationOid' belongs to a partitioned table - * then we issue a warning to mention these are not yet supported. + * itself will be rebuilt. * * The locks taken on parent tables and involved indexes are kept until the * transaction is committed, at which point a session lock is taken on each @@ -3010,13 +2996,6 @@ ReindexRelationConcurrently(Oid relationOid, int options) MemoryContextSwitchTo(oldcontext); break; } - case RELKIND_PARTITIONED_TABLE: - /* see reindex_relation() */ - ereport(WARNING, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX of partitioned tables is not yet implemented, skipping \"%s\"", - get_rel_name(relationOid)))); - return false; default: /* Return error if type of relation is not supported */ ereport(ERROR, @@ -3478,17 +3457,71 @@ ReindexRelationConcurrently(Oid relationOid, int options) } /* - * ReindexPartitionedIndex - * Reindex each child of the given partitioned index. - * - * Not yet implemented. + * ReindexPartitionedRel + * Reindex each child of the given partitioned relation. */ static void -ReindexPartitionedIndex(Relation parentIdx) +ReindexPartitionedRel(Oid reloid, int options, bool concurrent, bool isTopLevel) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("REINDEX is not yet implemented for partitioned indexes"))); + MemoryContext oldcontext, reindex_context; + List *inhoids; + ListCell *lc; + + /* + * This cannot run inside a user transaction block; if + * we were inside a transaction, then its commit- and + * start-transaction-command calls would not have the + * intended effect! + */ + PreventInTransactionBlock(isTopLevel, + "REINDEX of partitioned relations"); // XXX + + /* + * Create list of children in longlived context, since we process each + * child in a separate transaction + */ + reindex_context = AllocSetContextCreate(PortalContext, "Reindex", + ALLOCSET_DEFAULT_SIZES); + oldcontext = MemoryContextSwitchTo(reindex_context); + inhoids = find_all_inheritors(reloid, NoLock, NULL); + MemoryContextSwitchTo(oldcontext); + + foreach (lc, inhoids) + { + Oid inhrelid = lfirst_oid(lc); + + PopActiveSnapshot(); + CommitTransactionCommand(); + + StartTransactionCommand(); + /* functions in indexes may want a snapshot set */ + PushActiveSnapshot(GetTransactionSnapshot()); + + switch (get_rel_relkind(inhrelid)) + { + case RELKIND_PARTITIONED_INDEX: + case RELKIND_PARTITIONED_TABLE: + /* + * We have a full list of direct and indirect children, so skip + * partitioned relations and just handle their children. + */ + continue; + case RELKIND_INDEX: + reindex_index(inhrelid, false, get_rel_persistence(inhrelid), + options | REINDEXOPT_REPORT_PROGRESS); + break; + case RELKIND_RELATION: + (void) reindex_relation(inhrelid, + REINDEX_REL_PROCESS_TOAST | + REINDEX_REL_CHECK_CONSTRAINTS, + options | REINDEXOPT_REPORT_PROGRESS); + break; + } + } + + PopActiveSnapshot(); + CommitTransactionCommand(); + StartTransactionCommand(); } /* diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 9b0c376c8c..fd6bc65c18 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -926,10 +926,12 @@ standard_ProcessUtility(PlannedStmt *pstmt, switch (stmt->kind) { case REINDEX_OBJECT_INDEX: - ReindexIndex(stmt->relation, stmt->options, stmt->concurrent); + ReindexIndex(stmt->relation, stmt->options, + stmt->concurrent, isTopLevel); break; case REINDEX_OBJECT_TABLE: - ReindexTable(stmt->relation, stmt->options, stmt->concurrent); + ReindexTable(stmt->relation, stmt->options, + stmt->concurrent, isTopLevel); break; case REINDEX_OBJECT_SCHEMA: case REINDEX_OBJECT_SYSTEM: diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index c26a102b17..df32f5b201 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -34,8 +34,10 @@ extern ObjectAddress DefineIndex(Oid relationId, bool check_not_in_use, bool skip_build, bool quiet); -extern void ReindexIndex(RangeVar *indexRelation, int options, bool concurrent); -extern Oid ReindexTable(RangeVar *relation, int options, bool concurrent); +extern void ReindexIndex(RangeVar *indexRelation, int options, bool concurrent, + bool isTopLevel); +extern Oid ReindexTable(RangeVar *relation, int options, bool concurrent, + bool isTopLevel); extern void ReindexMultipleTables(const char *objectName, ReindexObjectType objectKind, int options, bool concurrent); extern char *makeObjectName(const char *name1, const char *name2, diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out index e3e6634d7e..61b4a30db4 100644 --- a/src/test/regress/expected/create_index.out +++ b/src/test/regress/expected/create_index.out @@ -2196,18 +2196,20 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind concur_reindex_part_index_0_2 | concur_reindex_part_index_0 | 2 (5 rows) --- REINDEX fails for partitioned indexes +-- REINDEX for partitioned indexes +REINDEX INDEX concur_reindex_part_index; +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +REINDEX INDEX concur_reindex_part_index_0; +REINDEX INDEX CONCURRENTLY concur_reindex_part_index_0; REINDEX INDEX concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; -ERROR: REINDEX is not yet implemented for partitioned indexes --- REINDEX is a no-op for partitioned tables +-- REINDEX for partitioned tables +REINDEX TABLE concur_reindex_part_10; +REINDEX TABLE CONCURRENTLY concur_reindex_part_10; +REINDEX TABLE concur_reindex_part_0; +REINDEX TABLE CONCURRENTLY concur_reindex_part_0; REINDEX TABLE concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes to reindex REINDEX TABLE CONCURRENTLY concur_reindex_part_10; -WARNING: REINDEX of partitioned tables is not yet implemented, skipping "concur_reindex_part_10" -NOTICE: table "concur_reindex_part_10" has no indexes that can be reindexed concurrently SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; relid | parentrelid | level diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql index f3667bacdc..a751dd5caa 100644 --- a/src/test/regress/sql/create_index.sql +++ b/src/test/regress/sql/create_index.sql @@ -903,12 +903,21 @@ CREATE INDEX concur_reindex_part_index_0_2 ON ONLY concur_reindex_part_0_2 (c1); ALTER INDEX concur_reindex_part_index_0 ATTACH PARTITION concur_reindex_part_index_0_2; SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; --- REINDEX fails for partitioned indexes +-- REINDEX for partitioned indexes +REINDEX INDEX concur_reindex_part_index; +REINDEX INDEX CONCURRENTLY concur_reindex_part_index; +REINDEX INDEX concur_reindex_part_index_0; +REINDEX INDEX CONCURRENTLY concur_reindex_part_index_0; REINDEX INDEX concur_reindex_part_index_10; REINDEX INDEX CONCURRENTLY concur_reindex_part_index_10; --- REINDEX is a no-op for partitioned tables +-- REINDEX for partitioned tables REINDEX TABLE concur_reindex_part_10; REINDEX TABLE CONCURRENTLY concur_reindex_part_10; +REINDEX TABLE concur_reindex_part_0; +REINDEX TABLE CONCURRENTLY concur_reindex_part_0; +REINDEX TABLE concur_reindex_part_10; +REINDEX TABLE CONCURRENTLY concur_reindex_part_10; + SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; -- REINDEX should preserve dependencies of partition tree. -- 2.17.0 --JYK4vJDZwFMowpUq Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-Allow-CREATE-INDEX-CONCURRENTLY-on-partitioned-ta.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH] cirrus/windows: ccache @ 2022-05-22 16:55 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-22 16:55 UTC (permalink / raw) https://community.chocolatey.org/packages/ccache https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com [email protected] 20211012083721.hvixq4pnh2pixr3j%40alap3.anarazel.de CA%2BhUKGLs6ENKL4w1o%2B1RpcS4VLyLXSUMWRYJVaiLBOJtarW25g%40mail.gmail.com https://cirrus-ci.com/task/5456827583299584 https://cirrus-ci.com/task/6019777536720896 ci-os-only: xindows, windows-meson-ninja, xindows-meson-msbuild --- .cirrus.yml | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index bf20f85f8ce..ed9e86557d7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -814,6 +814,9 @@ windows_template: &windows_template # currently have a tool for that... CIRRUS_ESCAPING_PROCESSES: 1 + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_LOGFILE: ccache.log + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' windows_container: @@ -828,12 +831,14 @@ windows_template: &windows_template set setup_additional_packages_script: | - REM choco install -y --no-progress ... - + choco install -y --no-progress ccache --version 4.6.1 + cp c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.6.1-windows-x86_64\ccache.exe \BuildTools\VC\Auxiliary\Build + cp c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.6.1-windows-x86_64\ccache.exe \ProgramData\chocolatey\bin\cl.exe task: <<: *windows_template name: Windows - Server 2019, VS 2019 - Homegrown + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows-mkvcbuild' configure_script: # copy errors out when using forward slashes @@ -901,14 +906,22 @@ task: - mkdir subprojects - meson wrap install lz4 - meson wrap install zlib - - meson setup --buildtype debug --backend ninja -Dcassert=true -Db_pch=true -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe build + - set CC=ccache.exe cl.exe + - meson setup --buildtype plain --backend ninja -Dcassert=true -Db_pch=false -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe -Dc_args="/Z7 /MDd" build + + ccache_cache: + folder: $CCACHE_DIR build_script: + - ccache.exe --zero-stats - vcvarsall x64 - ninja -C build + - ccache.exe --show-stats + + always: + upload_caches: ccache check_world_script: - - vcvarsall x64 - meson test --no-rebuild -C build on_failure: @@ -934,14 +947,21 @@ task: - mkdir subprojects - meson wrap install lz4 - meson wrap install zlib - - meson setup --buildtype debug --backend vs -Dcassert=true -Db_pch=true -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe build + - meson setup --buildtype plain --backend vs -Dcassert=true -Db_pch=false -Dssl=openssl -Dlz4=enabled -Dzlib=enabled -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=c:/windows/system32/tar.exe -Dc_args=/Z7 build + + ccache_cache: + folder: $CCACHE_DIR build_script: + - ccache.exe --zero-stats - vcvarsall x64 - - msbuild /p:UseMultiToolTask=true %MSBFLAGS% build\postgresql.sln + - msbuild /p:UseMultiToolTask=true /p:CLToolPath=c:\ProgramData\chocolatey\bin %MSBFLAGS% build\postgresql.sln + - ccache.exe --show-stats + + always: + upload_caches: ccache check_world_script: - - vcvarsall x64 - meson test --no-rebuild -C build always: -- 2.17.1 --CblX+4bnyfN0pR09-- ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 2/6] cirrus/windows: ccache @ 2022-05-26 03:05 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-26 03:05 UTC (permalink / raw) https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com https://cirrus-ci.com/task/5213936495099904 build 29sec , linux linux-meson ci-os-only: windows, windows-msvc --- .cirrus.tasks.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 5a2b64f64c2..7a3fcc36fae 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -536,6 +536,11 @@ task: env: TEST_JOBS: 8 # wild guess, data based value welcome + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_MAXSIZE: "500M" + CCACHE_SLOPPINESS: pch_defines,time_macros + CCACHE_DEPEND: 1 + # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That # prevents crash reporting from working unless binaries do SetErrorMode() # themselves. Furthermore, it appears that either python or, more likely, @@ -550,8 +555,11 @@ task: depends_on: SanityCheck only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' - setup_additional_packages_script: | - REM choco install -y --no-progress ... + setup_additional_packages_script: + - choco install -y --no-progress ccache --version 4.10.0 + + ccache_cache: + folder: $CCACHE_DIR setup_hosts_file_script: | echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts @@ -562,7 +570,8 @@ task: # Use /DEBUG:FASTLINK to avoid high memory usage during linking configure_script: | vcvarsall x64 - meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build + set CC=c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.10-windows-x86_64\ccache.exe cl.exe + meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7" build_script: | vcvarsall x64 @@ -571,6 +580,12 @@ task: REM without the pipe, exiting now if it fails, to avoid trying to run checks ninja -C build > nul + ccache --show-log-stats -v + ccache --show-stats -v + + always: + upload_caches: [ccache] + check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% -- 2.42.0 --Rvs4meNCoh6yKfx/ Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-WIP-ci-meson-allow-showing-only-failed-tests.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 12/19] cirrus/windows: ccache @ 2022-05-26 03:05 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-26 03:05 UTC (permalink / raw) https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com ci-os-only: windows --- .cirrus.yml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 02f04aa98bb..35e51453dfd 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -372,7 +372,15 @@ task: # disable file tracker, we're never going to rebuild, and it slows down the # build # -fileLoggerParameters1: write warnings to msbuild.warn.log. - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log /p:UseMultiToolTask=true /p:CLToolExe=pgccache.exe +#c:\ProgramData\chocolatey\bin\ccache.exe + + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_LOGFILE: ccache.log + #CCACHE_PREFIX: cl.exe + #CCACHE_PREFIX_CPP: cl.exe + CCACHE_COMPILERTYPE: msvc + CCACHE_COMPILER: cl.exe DebugInformationFormat: OldStyle @@ -403,8 +411,12 @@ task: powershell -Command get-psdrive -psprovider filesystem set - setup_additional_packages_script: | - REM choco install -y --no-progress ... + setup_additional_packages_script: + - choco install -y --no-progress ccache --version 4.6.1 + - cp c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.6.1-windows-x86_64\ccache.exe \ProgramData\chocolatey\bin\pgccache.exe + + ccache_cache: + folder: $CCACHE_DIR configure_script: # copy errors out when using forward slashes @@ -413,7 +425,13 @@ task: - perl src/tools/msvc/mkvcbuild.pl build_script: - vcvarsall x64 + - ccache --zero-stats - msbuild %MSBFLAGS% pgsql.sln + - ccache.exe --show-stats + + always: + upload_caches: ccache + tempinstall_script: # Installation on windows currently only completely works from src/tools/msvc - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install -- 2.17.1 --Sw7tCqrGA+HQ0/zt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0013-another-way-to-install-uri-regress-uri_regress.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 12/21] cirrus/windows: ccache @ 2022-05-26 03:05 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-26 03:05 UTC (permalink / raw) https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com ci-os-only: windows, windows-msvc --- .cirrus.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 02f04aa98bb..e5654bd5589 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -372,7 +372,11 @@ task: # disable file tracker, we're never going to rebuild, and it slows down the # build # -fileLoggerParameters1: write warnings to msbuild.warn.log. - MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log + MSBFLAGS: -m -verbosity:minimal "-consoleLoggerParameters:Summary;ForceNoAlign" /p:TrackFileAccess=false -nologo -fileLoggerParameters1:warningsonly;logfile=msbuild.warn.log /p:UseMultiToolTask=true /p:CLToolPath=c:\ProgramData\chocolatey\bin +#c:\ProgramData\chocolatey\bin\ccache.exe + + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_LOGFILE: ccache.log DebugInformationFormat: OldStyle @@ -403,8 +407,12 @@ task: powershell -Command get-psdrive -psprovider filesystem set - setup_additional_packages_script: | - REM choco install -y --no-progress ... + setup_additional_packages_script: + - choco install -y --no-progress ccache --version 4.6.1 + - cp c:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.6.1-windows-x86_64\ccache.exe \ProgramData\chocolatey\bin\cl.exe + + ccache_cache: + folder: $CCACHE_DIR configure_script: # copy errors out when using forward slashes @@ -414,6 +422,10 @@ task: build_script: - vcvarsall x64 - msbuild %MSBFLAGS% pgsql.sln + + always: + upload_caches: ccache + tempinstall_script: # Installation on windows currently only completely works from src/tools/msvc - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install -- 2.17.1 --STPqjqpCrtky8aYs Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0013-cirrus-ccache-disable-compression-and-show-stats.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 2/7] cirrus/windows: ccache @ 2022-05-26 03:05 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-26 03:05 UTC (permalink / raw) ccache 4.7 added support for depend mode with MSVC, and ccache v4.10 supports PCH, so this is now viable. https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com https://cirrus-ci.com/task/5213936495099904 build 29sec --- .cirrus.tasks.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 5a2b64f64c2..6ebbc1ccd81 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -536,6 +536,12 @@ task: env: TEST_JOBS: 8 # wild guess, data based value welcome + CC: c:\msys64\ucrt64\bin\ccache.exe cl.exe + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_MAXSIZE: "500M" + CCACHE_SLOPPINESS: pch_defines,time_macros + CCACHE_DEPEND: 1 + # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That # prevents crash reporting from working unless binaries do SetErrorMode() # themselves. Furthermore, it appears that either python or, more likely, @@ -553,6 +559,9 @@ task: setup_additional_packages_script: | REM choco install -y --no-progress ... + ccache_cache: + folder: $CCACHE_DIR + setup_hosts_file_script: | echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts @@ -562,7 +571,7 @@ task: # Use /DEBUG:FASTLINK to avoid high memory usage during linking configure_script: | vcvarsall x64 - meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build + meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7" build_script: | vcvarsall x64 @@ -571,6 +580,9 @@ task: REM without the pipe, exiting now if it fails, to avoid trying to run checks ninja -C build > nul + always: + upload_caches: [ccache] + check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% -- 2.42.0 --jviKHM0HgsS9J3Sv Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-WIP-ci-meson-allow-showing-only-failed-tests.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH 2/6] cirrus/windows: ccache @ 2022-05-26 03:05 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 7+ messages in thread From: Justin Pryzby @ 2022-05-26 03:05 UTC (permalink / raw) ccache 4.7 added support for depend mode with MSVC, and ccache v4.10 supports PCH, so this is now viable. https://www.postgresql.org/message-id/flat/20220522232606.GZ19626%40telsasoft.com https://cirrus-ci.com/task/5213936495099904 build 29sec --- .cirrus.tasks.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 5dd37e07aae..fb4ea3d666a 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -546,6 +546,12 @@ task: env: TEST_JOBS: 8 # wild guess, data based value welcome + CC: c:\msys64\ucrt64\bin\ccache.exe cl.exe + CCACHE_DIR: $CIRRUS_WORKING_DIR/.ccache + CCACHE_MAXSIZE: "500M" + CCACHE_SLOPPINESS: pch_defines,time_macros + CCACHE_DEPEND: 1 + # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That # prevents crash reporting from working unless binaries do SetErrorMode() # themselves. Furthermore, it appears that either python or, more likely, @@ -563,6 +569,9 @@ task: setup_additional_packages_script: | REM choco install -y --no-progress ... + ccache_cache: + folder: $CCACHE_DIR + setup_hosts_file_script: | echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts @@ -570,9 +579,10 @@ task: type c:\Windows\System32\Drivers\etc\hosts # Use /DEBUG:FASTLINK to avoid high memory usage during linking + # Use /Z7 to write separate debug files, to allow ccache to work configure_script: | vcvarsall x64 - meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build + meson setup build --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" -Dc_args="/Z7" build_script: | vcvarsall x64 @@ -581,6 +591,9 @@ task: REM without the pipe, exiting now if it fails, to avoid trying to run checks ninja -C build > nul + always: + upload_caches: [ccache] + check_world_script: | vcvarsall x64 meson test %MTEST_ARGS% --num-processes %TEST_JOBS% -- 2.42.0 --ndI8Dx/wEX5btThR Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0003-WIP-ci-meson-allow-showing-only-failed-tests.patch" ^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2022-05-26 03:05 UTC | newest] Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-06-07 20:59 [PATCH v4 1/3] Implement REINDEX of partitioned tables/indexes Justin Pryzby <[email protected]> 2022-05-22 16:55 [PATCH] cirrus/windows: ccache Justin Pryzby <[email protected]> 2022-05-26 03:05 [PATCH 2/6] cirrus/windows: ccache Justin Pryzby <[email protected]> 2022-05-26 03:05 [PATCH 12/19] cirrus/windows: ccache Justin Pryzby <[email protected]> 2022-05-26 03:05 [PATCH 12/21] cirrus/windows: ccache Justin Pryzby <[email protected]> 2022-05-26 03:05 [PATCH 2/7] cirrus/windows: ccache Justin Pryzby <[email protected]> 2022-05-26 03:05 [PATCH 2/6] cirrus/windows: ccache Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox