public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v1 2/6] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
7+ messages / 2 participants
[nested] [flat]
* [PATCH v1 2/6] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
---
.cirrus.yml | 306 ++++++++++++++++++++++++++++++++
.dockerignore | 3 +
ci/docker/linux_debian_bullseye | 9 +
ci/docker/windows_vs_2019 | 82 +++++++++
ci/freebsd_gcp_repartition.sh | 28 +++
ci/pg_ci_base.conf | 12 ++
ci/windows_build_config.pl | 3 +
7 files changed, 443 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 .dockerignore
create mode 100644 ci/docker/linux_debian_bullseye
create mode 100644 ci/docker/windows_vs_2019
create mode 100755 ci/freebsd_gcp_repartition.sh
create mode 100644 ci/pg_ci_base.conf
create mode 100644 ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..9f513f7d10a
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,306 @@
+env:
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+ # target to test, for all but windows
+ CHECK: check-world
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ CCACHE_MAXSIZE: "500M"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+
+
+task:
+ name: FreeBSD
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-freebsd-13-0
+ platform: freebsd
+ cpu: 2
+ memory: 2G
+ disk: 50
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ sysinfo_script:
+ - export || true
+ sysconfig_script:
+ - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+ repartition_script:
+ - ci/freebsd_gcp_repartition.sh
+ create_user_script:
+ - pw useradd postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+
+ configure_script:
+ - su postgres -c './configure --enable-cassert --enable-debug --enable-tap-tests --with-includes=/usr/local/include --with-libs=/usr/local/lib CC="ccache cc"'
+ build_script:
+ - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+ upload_caches:
+ - ccache
+
+ check_world_script:
+ - su postgres -c 'time gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+ on_failure:
+ core_script:
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ; echo dumping $corefile for $binary ; gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ; done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Linux
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-bullseye
+ platform: linux
+ cpu: 4
+ memory: 8G
+ nested_virtualization: false
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+ TIMEOUT_CMD: timeout -s KILL -v 25m
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - id
+ - uname -a
+ - cat /proc/cmdline
+ - lsblk
+ - ulimit -a -H
+ - ulimit -a -S
+ - export
+ sysconfig_script:
+ - useradd -m postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+ - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ - su postgres -c 'ulimit -l -H'
+ - su postgres -c 'ulimit -l -S'
+ - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+ configure_script:
+ - echo "COPT=-O0 -ggdb" > src/Makefile.custom
+ - su postgres -c './configure --enable-cassert --enable-debug --enable-tap-tests --with-tcl --with-python --with-perl --with-ldap --with-openssl --with-icu --with-llvm CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"'
+ build_script:
+ - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - su postgres -c 'ulimit -c unlimited ; ${TIMEOUT_CMD} make -s ${CHECK} ${CHECKFLAGS} -j8'
+
+ on_failure:
+ cores_script:
+ - for corefile in $(find /tmp/ -name '*.core' 2>/dev/null) ; do binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ; echo dumping $corefile for $binary ; gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ; done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: macOS
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+ osx_instance:
+ image: big-sur-base
+ env:
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ sysinfo_script:
+ - id
+ - export
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+ homebrew_cache:
+ folder: ${HOMEBREW_CACHE}
+ perl_cache:
+ folder: ~/perl5
+
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches:
+ - perl
+ core_install_script:
+ - sudo chmod 777 /cores
+ homebrew_install_script:
+ - brew install make coreutils ccache
+ upload_caches:
+ - homebrew
+
+ configure_script:
+ - ./configure --prefix=$HOME/install --enable-cassert --enable-debug --enable-tap-tests CC="ccache gcc" CFLAGS="-O0"
+ build_script:
+ - gmake -s -j12 && gmake -s -j12 -C contrib
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - ulimit -c unlimited
+ - ulimit -n 1024
+ - gtimeout -s KILL -v 20m gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ cores_script:
+ - for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ; done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Windows
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+ windows_container:
+ dockerfile: ci/docker/windows_vs_2019
+ cpu: 4
+ memory: 4G
+ env:
+ PROVE_FLAGS: -j10
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+
+ sysinfo_script:
+ - chcp
+ - systeminfo
+ - powershell -Command get-psdrive -psprovider filesystem
+ - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+ - set
+
+ configure_script:
+ - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ # Disable file tracker, we're never going to rebuild...
+ - msbuild -m /p:TrackFileAccess=false pgsql.sln
+ tempinstall_script:
+ - perl src\tools\msvc\install.pl tmp_install
+
+ check_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_script:
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+ isolationcheck_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ modulescheck_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ contribcheck_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ plcheck_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ stop_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+ subscriptioncheck_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+ authentication_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+ recoverycheck_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ bincheck_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ upgradecheck_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ ecpgcheck_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src\tools\msvc
+ - perl vcregress.pl ecpgcheck
+
+ always:
+ cat_dumps_script:
+ - cat crashlog.txt || true
+ dump_artifacts:
+ path: "crashlog.txt"
+ type: text/plain
+
+ on_failure:
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: CompilerWarnings
+ depends_on:
+ - Linux
+ # not run task count as a success, so we need to recheck Linux' condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ container:
+ dockerfile: ci/docker/linux_debian_bullseye
+ env:
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ setup_script:
+ - echo "COPT=-Werror" > src/Makefile.custom
+ - gcc -v
+ - clang -v
+ # gcc with asserts disabled
+ always:
+ gcc_warning_script:
+ - ./configure --cache gcc.cache CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # gcc with asserts enabled
+ always:
+ gcc_a_warning_script:
+ - ./configure --cache gcc.cache --enable-cassert CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts disabled
+ always:
+ clang_warning_script:
+ - ./configure --cache clang.cache CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts enabled
+ always:
+ clang_a_warning_script:
+ - ./configure --cache clang.cache --enable-cassert CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..3fceab2e97b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+# Ignore everything, except ci/
+*
+!ci/*
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..3c1d1a4583a
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,9 @@
+FROM debian:bullseye
+RUN apt-get -y update && apt-get -y upgrade && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
+ gcc g++ libreadline-dev flex bison make perl libipc-run-perl clang \
+ llvm-dev libperl-dev libpython3-dev tcl-dev libldap2-dev libicu-dev \
+ docbook-xml docbook-xsl fop libxml2-utils xsltproc krb5-admin-server \
+ krb5-kdc krb5-user slapd ldap-utils libssl-dev pkg-config locales-all \
+ liburing-dev python3-distutils ccache gdb && \
+ apt-get clean
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..e5b4810ac9e
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,82 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+RUN `
+ New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio' ; `
+ New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' -Name KeepDownloadedPayloads -Value 0 -PropertyType DWord
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdk... ; `
+ Start-Process -FilePath ".\windsdksetup.exe" `
+ -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+ -Wait ; `
+ `
+ Start-Process -FilePath msiexec.exe `
+ -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+ -Wait ; `
+ C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t\* -Force -Recurse ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+ Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger; `
+ setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m
+
+
+# Install perl and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's wy curl.exe is directly used here at least...
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -sSL -o perl.zip `
+ https://strawberryperl.com/download/5.30.3.1/strawberry-perl-5.30.3.1-64bit-portable.zip ; `
+ 7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+ `
+ curl.exe -sSL -o winflexbison.zip `
+ https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+ 7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+ Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+ Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m
+
+
+# Install visual studio
+#
+# Adding VS path to vcvarsall.bat so user of container doesn't need to know the full path
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+ Start-Process -Wait `
+ -FilePath c:\t\vs_buildtools.exe `
+ -ArgumentList `
+ '--quiet', '--wait', '--norestart', '--nocache', `
+ '--installPath', 'c:\BuildTools', `
+ '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+ '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348' ; `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+ Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\" ; `
+ setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m
diff --git a/ci/freebsd_gcp_repartition.sh b/ci/freebsd_gcp_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/freebsd_gcp_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..637e3cfb343
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,12 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions=10
+
+# Settings that make logs more useful
+log_line_prefix='%m [%p][%b][%v:%x] '
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..a8abfa2a02f
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,3 @@
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+1;
--
2.32.0.rc2
--ntohwnblr5nw7rqz
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v1-0003-Reference-test-binary-using-TESTDIR-in-001_libpq_.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v3 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
Author: Andres Freund <[email protected]>
Author: Thomas Munro <[email protected]>
Author: Melanie Plageman <[email protected]>
Reviewed-By: Melanie Plageman <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
.cirrus.yml | 456 ++++++++++++++++++++++++++++++++
.dockerignore | 7 +
ci/README | 50 ++++
ci/docker/linux_debian_bullseye | 67 +++++
ci/docker/windows_vs_2019 | 122 +++++++++
ci/gcp_freebsd_repartition.sh | 28 ++
ci/pg_ci_base.conf | 14 +
ci/windows_build_config.pl | 13 +
8 files changed, 757 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 .dockerignore
create mode 100644 ci/README
create mode 100644 ci/docker/linux_debian_bullseye
create mode 100644 ci/docker/windows_vs_2019
create mode 100755 ci/gcp_freebsd_repartition.sh
create mode 100644 ci/pg_ci_base.conf
create mode 100644 ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..7a068650206
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,456 @@
+# What files to preserve in case tests fail
+on_failure: &on_failure
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+env:
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+ # target to test, for all but windows
+ CHECK: check-world
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ CCACHE_MAXSIZE: "500M"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ PG_TEST_EXTRA: kerberos ldap ssl
+
+
+task:
+ name: FreeBSD
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-freebsd-13-0
+ platform: freebsd
+ cpu: 2
+ memory: 2G
+ disk: 50
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ sysinfo_script:
+ - export
+ sysconfig_script:
+ - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+ repartition_script:
+ - ci/gcp_freebsd_repartition.sh
+ create_user_script:
+ - pw useradd postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
+ --with-uuid=bsd \
+ \
+ --with-includes=/usr/local/include \
+ --with-libs=/usr/local/lib \
+ \
+ CC="ccache cc" CFLAGS="-O0 -ggdb"'
+ build_script:
+ - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+ upload_caches:
+ - ccache
+
+ # Use of script is to avoid make complaints about fcntl()
+ # https://savannah.gnu.org/bugs/?60774
+ test_world_script:
+ - su postgres -c 'time script test.log gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+ on_failure:
+ <<: *on_failure
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile;
+ done
+
+
+task:
+ name: Linux
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-bullseye
+ platform: linux
+ cpu: 4
+ memory: 2G
+ nested_virtualization: false
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - id
+ - uname -a
+ - cat /proc/cmdline
+ - lsblk
+ - ulimit -a -H
+ - ulimit -a -S
+ - export
+ sysconfig_script:
+ - useradd -m postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+ - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ - su postgres -c 'ulimit -l -H'
+ - su postgres -c 'ulimit -l -S'
+ - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-llvm \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-selinux \
+ --with-ssl=openssl \
+ --with-systemd \
+ --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CFLAGS="-O0 -ggdb" \
+ CXX="ccache g++" CXXFLAGS="-O0 -ggdb" \
+ CLANG="ccache clang" '
+ build_script:
+ - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+ upload_caches:
+ - ccache
+
+ test_world_script: |
+ su postgres -c '\
+ ulimit -c unlimited; \
+ make -s ${CHECK} ${CHECKFLAGS} -j8 \
+ '
+
+ on_failure:
+ <<: *on_failure
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ;
+ done
+
+
+task:
+ name: macOS
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+ osx_instance:
+ image: monterey-base
+ env:
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ sysinfo_script:
+ - id
+ - export
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+ homebrew_cache:
+ folder: ${HOMEBREW_CACHE}
+ perl_cache:
+ folder: ~/perl5
+
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches:
+ - perl
+ core_install_script:
+ - sudo chmod 777 /cores
+
+ # XXX: Could we instead install homebrew into a cached directory? The
+ # homebrew installation takes a good bit of time every time, even if the
+ # packages do not need to be downloaded.
+ homebrew_install_script: |
+ brew install \
+ ccache \
+ coreutils \
+ icu4c \
+ krb5 \
+ llvm \
+ lz4 \
+ make \
+ openldap \
+ openssl \
+ [email protected] \
+ tcl-tk
+
+ brew cleanup -s
+ upload_caches:
+ - homebrew
+
+ configure_script: |
+ LIBS="/usr/local/lib:$LIBS"
+ INCLUDES="/usr/local/include:$INCLUDES"
+
+ PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
+ INCLUDES="/usr/local/opt/icu4c/include:$INCLUDES"
+ LIBS="/usr/local/opt/icu4c/lib:$LIBS"
+
+ PKG_CONFIG_PATH="/usr/local/opt/krb5/lib/pkgconfig:$PKG_CONFIG_PATH"
+ INCLUDES="/usr/local/opt/krb5/include:$INCLUDES"
+ LIBS="/usr/local/opt/krb5/lib:$LIBS"
+
+ LIBS="/usr/local/opt/openldap/lib:$LIBS"
+ INCLUDES="/usr/local/opt/openldap/include:$INCLUDES"
+
+ INCLUDES="/usr/local/opt/openssl/include:$INCLUDES"
+ LIBS="/usr/local/opt/openssl/lib:$LIBS"
+
+ export PKG_CONFIG_PATH
+
+ ./configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-bonjour \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-lz4 \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/opt/tcl-tk/lib/ \
+ --with-uuid=e2fs \
+ \
+ --prefix=$HOME/install \
+ --with-includes="$INCLUDES" \
+ --with-libs="$LIBS" \
+ \
+ CC="ccache cc" CFLAGS="-O0 -ggdb" \
+ CXX="ccache c++" CXXFLAGS="-O0 -ggdb" \
+ CLANG="ccache /usr/local/opt/llvm/bin/ccache" \
+ LLVM_CONFIG=/usr/local/opt/llvm/bin/llvm-config \
+ PYTHON=/usr/local/opt/[email protected]/bin/python3
+ build_script:
+ - gmake -s -j12 && gmake -s -j12 -C contrib
+ upload_caches:
+ - ccache
+
+ test_world_script:
+ - ulimit -c unlimited
+ - ulimit -n 1024
+ # See freebsd use of script
+ - script test.log gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ <<: *on_failure
+ cores_script: |
+ for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do
+ lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ;
+ done
+
+
+task:
+ name: Windows
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+ windows_container:
+ dockerfile: ci/docker/windows_vs_2019
+ cpu: 4
+ memory: 4G
+ env:
+ PROVE_FLAGS: -j10
+ # The default cirrus working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+ # git's tar doesn't deal with drive letters, see
+ # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
+ TAR: "c:/windows/system32/tar.exe"
+ # Avoids port conflicts
+ PG_TEST_USE_UNIX_SOCKETS: 1
+ PG_REGRESS_SOCK_DIR: "c:/cirrus/"
+
+ sysinfo_script:
+ - chcp
+ - systeminfo
+ - powershell -Command get-psdrive -psprovider filesystem
+ - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+ - set
+
+ configure_script:
+ - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ # Disable file tracker, we're never going to rebuild...
+ - msbuild -m /p:TrackFileAccess=false pgsql.sln
+ tempinstall_script:
+ # Installation on windows currently only completely works from src\tools\msvc
+ - cd src\tools\msvc && perl .\install.pl %CIRRUS_WORKING_DIR%\tmp_install
+
+ test_check_parallel_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_script:
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+ test_plcheck_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ test_isolationcheck_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ test_modulescheck_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ test_contribcheck_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ stop_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+ test_ssl_script:
+ - set with_ssl=openssl
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\ssl\
+ test_subscriptioncheck_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+ test_authentication_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+ test_recoverycheck_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ test_bincheck_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ test_upgradecheck_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ test_ecpgcheck_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src\tools\msvc
+ - perl vcregress.pl ecpgcheck
+
+ on_failure: *on_failure
+
+
+task:
+ name: CompilerWarnings
+ depends_on:
+ - Linux
+ # task that did not run count as a success, so we need to recheck Linux'
+ # condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ container:
+ dockerfile: ci/docker/linux_debian_bullseye
+ env:
+ # Use larger ccache cache, as this task compiles with multiple compilers /
+ # flag combinations
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - gcc -v
+ - clang -v
+
+ setup_script:
+ - echo "COPT=-Werror" > src/Makefile.custom
+
+ ###
+ # Test that code can be built with gcc/clang without warnings
+ ###
+
+ # Trace probes have a history of getting accidentally broken. Use the
+ # different compilers to build with different combinations of dtrace on/off
+ # and cassert on/off.
+
+ # gcc, cassert off, dtrace on
+ always:
+ gcc_warning_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc" \
+ --enable-dtrace
+ make -s -j4 clean
+ time make -s -j4 world-bin
+ # gcc, cassert on, dtrace off
+ always:
+ gcc_a_warning_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc" \
+ --enable-cassert
+ make -s -j4 clean
+ time make -s -j4 world-bin
+ # clang, cassert off, dtrace off
+ always:
+ clang_warning_script: |
+ time ./configure \
+ --cache clang.cache CC="ccache clang"
+ make -s -j4 clean
+ time make -s -j4 world-bin
+ # clang, cassert on, dtrace on
+ always:
+ clang_a_warning_script: |
+ time ./configure \
+ --cache clang.cache CC="ccache clang" \
+ --enable-dtrace \
+ --enable-cassert
+ make -s -j4 clean
+ time make -s -j4 world-bin
+ # cross-compile to windows
+ always:
+ mingw_cross_warning_script: |
+ time ./configure \
+ --host=x86_64-w64-mingw32 \
+ --enable-cassert \
+ CC="ccache x86_64-w64-mingw32-gcc"
+ make -s -j4 clean
+ time make -s -j4 world-bin
+
+ ###
+ # Verify docs can be built
+ ###
+
+ # XXX: Only do this if there have been changes in doc/ since last build
+ always:
+ docs_build_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc"
+ time make -s -j4 -C doc
+
+ always:
+ upload_caches:
+ - ccache
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..f27b14d5c26
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,7 @@
+# Ignore everything, except ci/
+#
+
+# cirrus-ci uses the whole repository as context when building dockerfiles,
+# and the dockerfiles should not reference anything outside of ci/
+*
+!ci/*
diff --git a/ci/README b/ci/README
new file mode 100644
index 00000000000..fda6eb39fb0
--- /dev/null
+++ b/ci/README
@@ -0,0 +1,50 @@
+Postgres Continuous Integration (CI)
+====================================
+
+Postgres has two forms of CI:
+
+1) All supported branches in the main postgres repository are continuously
+ tested via the the the buildfarm. As this covers only the main repository,
+ it cannot be used during development of features.
+
+ For details see https://buildfarm.postgresql.org/
+
+2) For not yet merged development work, CI can be enabled for some git hosting
+ providers. This allows to test patches on a number of platforms before they
+ are merged (or even submitted).
+
+
+Configuring CI on personal repositories
+=======================================
+
+Currently postgres contains CI support utilizing cirrus-ci. cirrus-ci
+currently is only available for github.
+
+
+Enabling cirrus-ci in a github repository
+=========================================
+
+To enable cirrus-ci on a repository, go to
+https://github.com/marketplace/cirrus-ci and select "Public
+Repositories". Then "Install it for free" and "Complete order". The next page
+allows to configure which repositories cirrus-ci has access too. Choose the
+relevant repository and "Install".
+
+See also https://cirrus-ci.org/guide/quick-start/
+
+Once enabled on a repository, future commits and pull-requests in that
+repository will automatically trigger CI builds. These are visible from the
+commit history / PRs, and can also be viewed in the cirrus-ci UI at
+https://cirrus-ci.com/github/<username>/<reponame>/
+
+
+Controlling CI via commit messages
+==================================
+
+The behavior of CI can be controlled by special content in commit
+messages. Currently the following controls are available:
+
+- ci-os-only: {(freebsd|linux|macos|windows)}
+
+ Only runs CI on operating systems specified. This can be useful when
+ addressing portability issues affecting only a subset of platforms.
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..bf48509c9e4
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,67 @@
+FROM debian:bullseye
+RUN \
+ export DEBIAN_FRONTEND=noninteractive && \
+ apt-get -y update && \
+ apt-get -y upgrade --no-install-recommends && \
+ apt-get -y install --no-install-recommends \
+ \
+ build-essential \
+ gdb \
+ make \
+ git \
+ meson \
+ perl \
+ pkg-config \
+ \
+ bison \
+ ccache \
+ clang \
+ flex \
+ g++ \
+ gcc \
+ gettext \
+ \
+ libio-pty-perl \
+ libipc-run-perl \
+ python3-distutils \
+ \
+ libicu-dev \
+ libkrb5-*-heimdal \
+ libkrb5-dev \
+ libldap2-dev \
+ liblz4-dev \
+ libossp-uuid-dev \
+ libpam-dev \
+ libperl-dev \
+ libpython3-dev \
+ libreadline-dev \
+ libselinux*-dev \
+ libssl-dev \
+ libsystemd-dev \
+ liburing-dev \
+ libxml2-dev \
+ libxslt1-dev \
+ llvm-dev \
+ python3-dev \
+ systemtap-sdt-dev \
+ tcl-dev \
+ uuid-dev \
+ \
+ docbook-xml \
+ docbook-xsl \
+ fop \
+ libxml2-utils \
+ xsltproc \
+ \
+ krb5-admin-server \
+ krb5-kdc \
+ krb5-user \
+ ldap-utils \
+ locales-all \
+ slapd \
+ \
+ g++-mingw-w64-x86-64-win32 \
+ gcc-mingw-w64-x86-64-win32 \
+ libz-mingw-w64-dev \
+ && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..5e0ff91ab6e
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,122 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated (&bloated) now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdk... ; `
+ echo 'starting sdk installation (for debugger)' ; `
+ Start-Process -Wait -FilePath ".\windsdksetup.exe" `
+ -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+ ; `
+ `
+ Start-Process -Wait -FilePath msiexec.exe `
+ -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+ ; `
+ C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+ `
+ Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+ Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install perl, python, flex and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's why curl.exe is directly used here at
+# least...
+#
+# Using perl 5.26.3.1 for now, as newer versions don't currently work
+# correctly for plperl
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ echo 'adding to path, before setup below, so changes are not overwritten' ; `
+ setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o perl.zip `
+ https://strawberryperl.com/download/5.26.3.1/strawberry-perl-5.26.3.1-64bit-portable.zip ; `
+ echo 'installing perl' ; `
+ 7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+ `
+ curl.exe -sSL -o python.exe https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe ; `
+ echo 'installing python' ; `
+ Start-Process -Wait -FilePath ".\python.exe" `
+ -ArgumentList `
+ '/quiet', 'SimpleInstall=1', 'PrependPath=1', 'CompileAll=1', `
+ 'TargetDir=c:\python\', 'InstallAllUsers=1', 'Shortcuts=0', `
+ 'Include_docs=0', 'Include_tcltk=0', 'Include_tests=0' `
+ ; `
+ `
+ curl.exe -sSL -o winflexbison.zip `
+ https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+ echo 'installing winflexbison' ; `
+ 7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+ Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+ Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install openssl
+#
+# This is one of the windows binararies referenced by the openssl wiki. See
+# https://www.openssl.org/community/binaries.html and
+# https://wiki.openssl.org/index.php/Binaries
+#
+# It might be nicer to switch to the openssl built as part of curl-for-win,
+# but recent releases only build openssl 3, and that still seems troublesome
+# on windows,
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -o openssl-setup.exe -sSL https://slproweb.com/download/Win64OpenSSL-1_1_1L.exe ; `
+ echo 'staring openssl installation' ; `
+ Start-Process -Wait -FilePath ".\openssl-setup.exe" `
+ -ArgumentList '/DIR=c:\openssl\1.1.1l\ /VERYSILENT /SP- /SUPPRESSMSGBOXES' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install visual studio
+#
+# Adding path to vcvarsall.bat to PATH so user of container dont't need to
+# know the full path
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+ echo 'starting visual studio installation' ; `
+ Start-Process -Wait `
+ -FilePath c:\t\vs_buildtools.exe `
+ -ArgumentList `
+ '--quiet', '--wait', '--norestart', '--nocache', `
+ '--installPath', 'c:\BuildTools', `
+ '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+ '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+ Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\"
diff --git a/ci/gcp_freebsd_repartition.sh b/ci/gcp_freebsd_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/gcp_freebsd_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..d8faa9c26c1
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,14 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions = 10
+
+# Settings that make logs more useful
+log_autovacuum_min_duration = 0
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
+log_line_prefix = '%m [%p][%b] %q[%a][%v:%x] '
+log_lock_waits = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..ba82b13d69a
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,13 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+
+$config->{"openssl"} = "c:/openssl/1.1.1l/";
+$config->{"perl"} = "c:/strawberry/perl/";
+$config->{"python"} = "c:/python/";
+
+1;
--
2.34.0
--rbj3wvz2ukg3lhpx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v3-0002-windows-Improve-crash-assert-exception-handling.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v4 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
Author: Andres Freund <[email protected]>
Author: Thomas Munro <[email protected]>
Author: Melanie Plageman <[email protected]>
Reviewed-By: Melanie Plageman <[email protected]>
Reviewed-By: Justin Pryzby <[email protected]>
Reviewed-By: Thomas Munro <[email protected]>
Reviewed-By: Peter Eisentraut <[email protected]>
Discussion: https://postgr.es/m/[email protected]
---
.cirrus.yml | 526 ++++++++++++++++++++++++
src/tools/ci/README | 63 +++
src/tools/ci/cores_backtrace.sh | 50 +++
src/tools/ci/gcp_freebsd_repartition.sh | 28 ++
src/tools/ci/pg_ci_base.conf | 14 +
src/tools/ci/windows_build_config.pl | 13 +
6 files changed, 694 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 src/tools/ci/README
create mode 100755 src/tools/ci/cores_backtrace.sh
create mode 100755 src/tools/ci/gcp_freebsd_repartition.sh
create mode 100644 src/tools/ci/pg_ci_base.conf
create mode 100644 src/tools/ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..4116e0155dc
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,526 @@
+# CI configuration file for CI utilizing cirrus-ci.org
+#
+# See src/tools/ci/README for information.
+
+env:
+ # Source of images / containers
+ GCP_PROJECT: pg-ci-images
+ IMAGE_PROJECT: $GCP_PROJECT
+ CONTAINER_REPO: us-docker.pkg.dev/${GCP_PROJECT}/ci
+
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+
+ CCACHE_MAXSIZE: "500M"
+
+ # target to test, for all but windows
+ PROVE_FLAGS: --timer
+ CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
+ PG_TEST_EXTRA: kerberos ldap ssl
+
+
+# What files to preserve in case tests fail
+on_failure: &on_failure
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: FreeBSD - 13
+
+ env:
+ # FreeBSD on GCP is slow when running with larger number of CPUS /
+ # jobs. Using one more job than cpus seems to work best.
+ CPUS: 2
+ BUILD_JOBS: 3
+ TEST_JOBS: 3
+
+ CCACHE_DIR: /tmp/ccache_dir
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+
+ compute_engine_instance:
+ image_project: $IMAGE_PROJECT
+ image: family/pg-ci-freebsd-13-0
+ platform: freebsd
+ cpu: $CPUS
+ memory: 2G
+ disk: 50
+
+ sysinfo_script: |
+ id
+ uname -a
+ ulimit -a -H
+ ulimit -a -S
+ export
+
+ ccache_cache:
+ folder: $CCACHE_DIR
+ # Workaround around performance issues due to 32KB block size
+ repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
+ create_user_script: |
+ pw useradd postgres
+ chown -R postgres:postgres .
+ mkdir -p ${CCACHE_DIR}
+ chown -R postgres:postgres ${CCACHE_DIR}
+ setup_cores_script: |
+ mkdir -m 770 /tmp/cores
+ chown root:postgres /tmp/cores
+ sysctl kern.corefile='/tmp/cores/%N.%P.core'
+
+ # NB: Intentionally building without --with-llvm. The freebsd image size is
+ # already large enough to make task startup slow, and freebsd already takes
+ # longer than other platforms except for windows.
+ configure_script: |
+ su postgres <<-EOF
+ ./configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
+ --with-uuid=bsd \
+ \
+ --with-includes=/usr/local/include \
+ --with-libs=/usr/local/lib \
+ \
+ CC="ccache cc" CFLAGS="-O0 -ggdb"
+ EOF
+ build_script: su postgres -c "gmake -s -j${BUILD_JOBS} world-bin"
+ upload_caches: ccache
+
+ # Use of script is to avoid make -Otarget complaints about fcntl() on platforms
+ # without support for locking pipes. See also https://savannah.gnu.org/bugs/?60774
+ # script uses a pseudo-tty, which do support locking.
+ test_world_script:
+ - su postgres -c "time script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}"
+
+ on_failure:
+ <<: *on_failure
+ cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
+
+
+# This is shared between the task running the linux tests and the
+# CompilerWarnings task
+LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
+ --with-gssapi
+ --with-icu
+ --with-ldap
+ --with-libxml
+ --with-libxslt
+ --with-llvm
+ --with-lz4
+ --with-pam
+ --with-perl
+ --with-python
+ --with-selinux
+ --with-ssl=openssl
+ --with-systemd
+ --with-tcl --with-tclconfig=/usr/lib/tcl8.6/
+ --with-uuid=e2fs
+
+task:
+ name: Linux - Debian Bullseye
+
+ env:
+ CPUS: 4
+ BUILD_JOBS: 4
+ TEST_JOBS: 8 # experimentally derived to be a decent choice
+
+ CCACHE_DIR: /tmp/ccache_dir
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+
+ LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+
+ compute_engine_instance:
+ image_project: $IMAGE_PROJECT
+ image: family/pg-ci-bullseye
+ platform: linux
+ cpu: $CPUS
+ memory: 2G
+
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+
+ sysinfo_script: |
+ id
+ uname -a
+ cat /proc/cmdline
+ lsblk
+ ulimit -a -H
+ ulimit -a -S
+ export
+ create_user_script: |
+ useradd -m postgres
+ chown -R postgres:postgres .
+ mkdir -p ${CCACHE_DIR}
+ chown -R postgres:postgres ${CCACHE_DIR}
+ echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ su postgres -c 'ulimit -l -H'
+ su postgres -c 'ulimit -l -S'
+ setup_cores_script: |
+ mkdir -m 770 /tmp/cores
+ chown root:postgres /tmp/cores
+ sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'
+
+ configure_script: |
+ su postgres <<-EOF
+ ./configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ ${LINUX_CONFIGURE_FEATURES} \
+ \
+ CC="ccache gcc" CFLAGS="-O0 -ggdb" \
+ CXX="ccache g++" CXXFLAGS="-O0 -ggdb" \
+ CLANG="ccache clang"
+ EOF
+ build_script: su postgres -c 'make -s -j${BUILD_JOBS} world-bin'
+ upload_caches: ccache
+
+ test_world_script: |
+ su postgres <<-EOF
+ ulimit -c unlimited # default is 0
+ make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
+ EOF
+
+ on_failure:
+ <<: *on_failure
+ cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
+
+
+task:
+ name: macOS - Monterey
+
+ env:
+ CPUS: 12 # always get that much for cirrusci macOS instances
+ BUILD_JOBS: $CPUS
+ TEST_JOBS: $CPUS # already fast enough to not be worth tuning
+
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+
+ osx_instance:
+ image: monterey-base
+
+ sysinfo_script: |
+ id
+ export
+ ulimit -a -H
+ ulimit -a -S
+
+ setup_cores_script:
+ - mkdir ${HOME}/cores
+ - sudo sysctl kern.corefile="${HOME}/cores/core.%P"
+
+ perl_cache:
+ folder: ~/perl5
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches: perl
+
+
+ # XXX: Could we instead install homebrew into a cached directory? The
+ # homebrew installation takes a good bit of time every time, even if the
+ # packages do not need to be downloaded.
+ homebrew_cache:
+ folder: $HOMEBREW_CACHE
+ homebrew_install_script: |
+ brew install \
+ ccache \
+ icu4c \
+ krb5 \
+ llvm \
+ lz4 \
+ make \
+ openldap \
+ openssl \
+ python \
+ tcl-tk
+
+ brew cleanup -s # to reduce cache size
+ upload_caches: homebrew
+
+ ccache_cache:
+ folder: $CCACHE_DIR
+ configure_script: |
+ LIBS="/usr/local/lib:${LIBS}"
+ INCLUDES="/usr/local/include:${INCLUDES}"
+
+ brewpath=/usr/local/opt
+ for package in icu4c krb5 openldap openssl; do
+ PKG_CONFIG_PATH="${brewpath}/${package}/lib/pkgconfig:${PKG_CONFIG_PATH}"
+ INCLUDES="${brewpath}/${package}/include:${INCLUDES}"
+ LIBS="${brewpath}/${package}/lib:${LIBS}"
+ done
+
+ export PKG_CONFIG_PATH
+
+ ./configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-bonjour \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-lz4 \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=${brewpath}/tcl-tk/lib/ \
+ --with-uuid=e2fs \
+ \
+ --prefix=${HOME}/install \
+ --with-includes="${INCLUDES}" \
+ --with-libs="${LIBS}" \
+ \
+ CC="ccache cc" CFLAGS="-O0 -ggdb" \
+ CXX="ccache c++" CXXFLAGS="-O0 -ggdb" \
+ CLANG="ccache ${brewpath}/llvm/bin/ccache" \
+ LLVM_CONFIG=${brewpath}/llvm/bin/llvm-config \
+ PYTHON=python3
+ build_script: gmake -s -j${BUILD_JOBS} world-bin
+ upload_caches: ccache
+
+ test_world_script: |
+ ulimit -c unlimited # default is 0
+ ulimit -n 1024 # default is 256, pretty low
+ # See freebsd use of script for explanation
+ script test.log gmake -s -j${TEST_JOBS} ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ <<: *on_failure
+ cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
+
+
+task:
+ name: Windows - Server 2019, VS 2019
+
+ env:
+ # Half the allowed per-user CPU cores
+ CPUS: 4
+
+ # our windows infrastructure doesn't have test concurrency above the level
+ # of a single vcregress test target. Because of that it makes sense to use
+ # concurrency within prove. For other targets it's not really useful
+ # because two sources of concurrency can overload machines.
+ #
+ # The concrete choice of 10 is based on a small bit of experimentation and
+ # likely can be improved upon further.
+ PROVE_FLAGS: -j10 --timer
+
+ # The default cirrus working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+ # git's tar doesn't deal with drive letters, see
+ # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
+ TAR: "c:/windows/system32/tar.exe"
+ # Avoids port conflicts between concurrent tap test runs
+ PG_TEST_USE_UNIX_SOCKETS: 1
+ PG_REGRESS_SOCK_DIR: "c:/cirrus/"
+ # Use parallelism, disable file tracker, we're never going to rebuild...
+ MSBFLAGS: -m -verbosity:minimal /p:TrackFileAccess=false
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+
+ windows_container:
+ image: $CONTAINER_REPO/windows_ci_vs_2019:latest
+ cpu: $CPUS
+ memory: 4G
+
+ sysinfo_script: |
+ chcp
+ systeminfo
+ powershell -Command get-psdrive -psprovider filesystem
+ set
+
+ configure_script:
+ # copy errors out when using forward slashes
+ - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ - msbuild %MSBFLAGS% pgsql.sln
+ tempinstall_script:
+ # Installation on windows currently only completely works from src/tools/msvc
+ - cd src/tools/msvc && perl install.pl %CIRRUS_WORKING_DIR%/tmp_install
+
+ test_regress_parallel_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_script:
+ # paths to binaries need backslashes
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check/db -l tmp_check/initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check/db/postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check/db -l tmp_check/postmaster.log
+ test_pl_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ test_isolation_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ test_modules_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ test_contrib_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ stop_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check/db -l tmp_check/postmaster.log
+ test_ssl_script:
+ - set with_ssl=openssl
+ - perl src/tools/msvc/vcregress.pl taptest ./src/test/ssl/
+ test_subscription_script:
+ - perl src/tools/msvc/vcregress.pl taptest ./src/test/subscription/
+ test_authentication_script:
+ - perl src/tools/msvc/vcregress.pl taptest ./src/test/authentication/
+ test_recovery_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ test_bin_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ test_pg_upgrade_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ test_ecpg_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src/tools/msvc
+ - perl vcregress.pl ecpgcheck
+
+ on_failure: *on_failure
+
+
+task:
+ name: CompilerWarnings
+
+ # To limit unneccessary work only run this once normal linux test succeeded
+ depends_on:
+ - Linux - Debian Bullseye
+
+ env:
+ CPUS: 4
+ BUILD_JOBS: 4
+
+ # Use larger ccache cache, as this task compiles with multiple compilers /
+ # flag combinations
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
+
+ # task that did not run count as a success, so we need to recheck Linux'
+ # condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+
+ container:
+ image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+
+ sysinfo_script: |
+ id
+ uname -a
+ ulimit -a -H
+ ulimit -a -S
+ gcc -v
+ clang -v
+ export
+
+ ccache_cache:
+ folder: $CCACHE_DIR
+
+ ###
+ # Test that code can be built with gcc/clang without warnings
+ ###
+
+ setup_script: echo "COPT=-Werror" > src/Makefile.custom
+
+ # Trace probes have a history of getting accidentally broken. Use the
+ # different compilers to build with different combinations of dtrace on/off
+ # and cassert on/off.
+
+ # gcc, cassert off, dtrace on
+ always:
+ gcc_warning_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc" \
+ --enable-dtrace \
+ ${LINUX_CONFIGURE_FEATURES}
+ make -s -j${BUILD_JOBS} clean
+ time make -s -j${BUILD_JOBS} world-bin
+ # gcc, cassert on, dtrace off
+ always:
+ gcc_a_warning_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc" \
+ --enable-cassert \
+ ${LINUX_CONFIGURE_FEATURES}
+ make -s -j${BUILD_JOBS} clean
+ time make -s -j${BUILD_JOBS} world-bin
+ # clang, cassert off, dtrace off
+ always:
+ clang_warning_script: |
+ time ./configure \
+ --cache clang.cache CC="ccache clang" \
+ ${LINUX_CONFIGURE_FEATURES}
+ make -s -j${BUILD_JOBS} clean
+ time make -s -j${BUILD_JOBS} world-bin
+ # clang, cassert on, dtrace on
+ always:
+ clang_a_warning_script: |
+ time ./configure \
+ --cache clang.cache CC="ccache clang" \
+ --enable-cassert \
+ --enable-dtrace \
+ ${LINUX_CONFIGURE_FEATURES}
+ make -s -j${BUILD_JOBS} clean
+ time make -s -j${BUILD_JOBS} world-bin
+ # cross-compile to windows
+ always:
+ mingw_cross_warning_script: |
+ time ./configure \
+ --host=x86_64-w64-mingw32 \
+ --enable-cassert \
+ CC="ccache x86_64-w64-mingw32-gcc"
+ make -s -j${BUILD_JOBS} clean
+ time make -s -j${BUILD_JOBS} world-bin
+
+ ###
+ # Verify docs can be built
+ ###
+
+ # XXX: Only do this if there have been changes in doc/ since last build
+ always:
+ docs_build_script: |
+ time ./configure \
+ --cache gcc.cache CC="ccache gcc"
+ time make -s -j${BUILD_JOBS} -C doc
+
+ always:
+ upload_caches: ccache
diff --git a/src/tools/ci/README b/src/tools/ci/README
new file mode 100644
index 00000000000..88487e07561
--- /dev/null
+++ b/src/tools/ci/README
@@ -0,0 +1,63 @@
+Postgres Continuous Integration (CI)
+====================================
+
+Postgres has two forms of CI:
+
+1) All supported branches in the main postgres repository are continuously
+ tested via the the the buildfarm. As this covers only the main repository,
+ it cannot be used during development of features.
+
+ For details see https://buildfarm.postgresql.org/
+
+2) For not yet merged development work, CI can be enabled for some git hosting
+ providers. This allows developers to test patches on a number of platforms
+ before they are merged (or even submitted).
+
+
+Configuring CI on personal repositories
+=======================================
+
+Currently postgres contains CI support utilizing cirrus-ci. cirrus-ci
+currently is only available for github.
+
+
+Enabling cirrus-ci in a github repository
+=========================================
+
+To enable cirrus-ci on a repository, go to
+https://github.com/marketplace/cirrus-ci and select "Public
+Repositories". Then "Install it for free" and "Complete order". The next page
+allows to configure which repositories cirrus-ci has access too. Choose the
+relevant repository and "Install".
+
+See also https://cirrus-ci.org/guide/quick-start/
+
+Once enabled on a repository, future commits and pull-requests in that
+repository will automatically trigger CI builds. These are visible from the
+commit history / PRs, and can also be viewed in the cirrus-ci UI at
+https://cirrus-ci.com/github/<username>/<reponame>/
+
+
+Controlling CI via commit messages
+==================================
+
+The behavior of CI can be controlled by special content in commit
+messages. Currently the following controls are available:
+
+- ci-os-only: {(freebsd|linux|macos|windows)}
+
+ Only runs CI on operating systems specified. This can be useful when
+ addressing portability issues affecting only a subset of platforms.
+
+
+Images used for CI
+==================
+
+To keep CI times tolerable, most platforms use pre-generated images. Some
+platforms use containers, others use full VMs. Images for both are generated
+separately from CI runs, otherwise each git repository that is being tested
+would need to build its own set of containers, which would be wasteful (both
+in space and time.
+
+These images are built, on a daily basis, from the specifications in
+github.com/anarazel/pg-vm-images/
diff --git a/src/tools/ci/cores_backtrace.sh b/src/tools/ci/cores_backtrace.sh
new file mode 100755
index 00000000000..549e5afce2b
--- /dev/null
+++ b/src/tools/ci/cores_backtrace.sh
@@ -0,0 +1,50 @@
+#! /bin/sh
+
+if [ $# -ne 2 ]; then
+ echo "cores_backtrace.sh <os> <directory>"
+ exit 1
+fi
+
+os=$1
+directory=$2
+
+case $os in
+ freebsd|linux|macos)
+ ;;
+ *)
+ echo "unsupported operating system ${os}"
+ exit 1
+ ;;
+esac
+
+first=1
+for corefile in $(find "$directory" -type f) ; do
+ if [ "$first" -eq 1 ]; then
+ first=0
+ else
+ # to make it easier to separate the different crash reports
+ echo -e '\n\n'
+ fi
+
+ if [ "$os" = 'macos' ]; then
+ lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit'
+ else
+ auxv=$(gdb --quiet --core ${corefile} --batch -ex 'info auxv' 2>/dev/null)
+ if [ $? -ne 0 ]; then
+ echo "could not process ${corefile}"
+ continue
+ fi
+
+ if [ "$os" = 'freebsd' ]; then
+ binary=$(echo "$auxv" | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+ elif [ "$os" = 'linux' ]; then
+ binary=$(echo "$auxv" | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g")
+ else
+ echo 'should not get here'
+ exit 1
+ fi
+
+ echo "dumping ${corefile} for ${binary}"
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" "$binary" "$corefile" 2>/dev/null
+ fi
+done
diff --git a/src/tools/ci/gcp_freebsd_repartition.sh b/src/tools/ci/gcp_freebsd_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/src/tools/ci/gcp_freebsd_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/src/tools/ci/pg_ci_base.conf b/src/tools/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..d8faa9c26c1
--- /dev/null
+++ b/src/tools/ci/pg_ci_base.conf
@@ -0,0 +1,14 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions = 10
+
+# Settings that make logs more useful
+log_autovacuum_min_duration = 0
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
+log_line_prefix = '%m [%p][%b] %q[%a][%v:%x] '
+log_lock_waits = true
diff --git a/src/tools/ci/windows_build_config.pl b/src/tools/ci/windows_build_config.pl
new file mode 100644
index 00000000000..b0d4360c748
--- /dev/null
+++ b/src/tools/ci/windows_build_config.pl
@@ -0,0 +1,13 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+
+$config->{"openssl"} = "c:/openssl/1.1/";
+$config->{"perl"} = "c:/strawberry/$ENV{DEFAULT_PERL_VERSION}/perl/";
+$config->{"python"} = "c:/python/";
+
+1;
--
2.34.0
--mezqkkzp3cvymuis
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v4-0002-windows-Improve-crash-assert-exception-handling.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v2 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
---
.cirrus.yml | 388 ++++++++++++++++++++++++++++++++
.dockerignore | 3 +
ci/README | 36 +++
ci/docker/linux_debian_bullseye | 13 ++
ci/docker/windows_vs_2019 | 111 +++++++++
ci/freebsd_gcp_repartition.sh | 28 +++
ci/pg_ci_base.conf | 12 +
ci/windows_build_config.pl | 13 ++
8 files changed, 604 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 .dockerignore
create mode 100644 ci/README
create mode 100644 ci/docker/linux_debian_bullseye
create mode 100644 ci/docker/windows_vs_2019
create mode 100755 ci/freebsd_gcp_repartition.sh
create mode 100644 ci/pg_ci_base.conf
create mode 100644 ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..b3321a91ae6
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,388 @@
+env:
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+ # target to test, for all but windows
+ CHECK: check-world
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ CCACHE_MAXSIZE: "500M"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ PG_TEST_EXTRA: kerberos ldap ssl
+
+
+task:
+ name: FreeBSD
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-freebsd-13-0
+ platform: freebsd
+ cpu: 2
+ memory: 2G
+ disk: 50
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ sysinfo_script:
+ - export || true
+ sysconfig_script:
+ - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+ repartition_script:
+ - ci/freebsd_gcp_repartition.sh
+ create_user_script:
+ - pw useradd postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
+ --with-uuid=bsd \
+ \
+ --with-includes=/usr/local/include --with-libs=/usr/local/lib \
+ CC="ccache cc"'
+ build_script:
+ - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - su postgres -c 'time gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Linux
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-bullseye
+ platform: linux
+ cpu: 4
+ memory: 2G
+ nested_virtualization: false
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - id
+ - uname -a
+ - cat /proc/cmdline
+ - lsblk
+ - ulimit -a -H
+ - ulimit -a -S
+ - export
+ sysconfig_script:
+ - useradd -m postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+ - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ - su postgres -c 'ulimit -l -H'
+ - su postgres -c 'ulimit -l -S'
+ - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-llvm \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-systemd \
+ --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" CFLAGS="-O0 -ggdb"'
+ build_script:
+ - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script: |
+ su postgres -c '\
+ ulimit -c unlimited; \
+ make -s ${CHECK} ${CHECKFLAGS} -j8 \
+ '
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: macOS
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+ osx_instance:
+ image: big-sur-base
+ env:
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ sysinfo_script:
+ - id
+ - export
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+ homebrew_cache:
+ folder: ${HOMEBREW_CACHE}
+ perl_cache:
+ folder: ~/perl5
+
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches:
+ - perl
+ core_install_script:
+ - sudo chmod 777 /cores
+ homebrew_install_script:
+ - brew install make coreutils ccache icu4c lz4 tcl-tk openldap
+ upload_caches:
+ - homebrew
+
+ configure_script: |
+ LIBS="/usr/local/lib:$LIBS"
+ INCLUDES="/usr/local/include:$INCLUDES"
+
+ INCLUDES="/usr/local/opt/openssl/include:$INCLUDES"
+ LIBS="/usr/local/opt/openssl/lib:$LIBS"
+
+ PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
+ INCLUDES="/usr/local/opt/icu4c/include:$INCLUDES"
+ LIBS="/usr/local/opt/icu4c/lib:$LIBS"
+
+ LIBS="/usr/local/opt/openldap/lib:$LIBS"
+ INCLUDES="/usr/local/opt/openldap/include:$INCLUDES"
+
+ export PKG_CONFIG_PATH
+
+ ./configure \
+ --prefix=$HOME/install \
+ --with-includes="$INCLUDES" \
+ --with-libs="$LIBS" \
+ \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/opt/tcl-tk/lib/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CFLAGS="-O0 -ggdb"
+ build_script:
+ - gmake -s -j12 && gmake -s -j12 -C contrib
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - ulimit -c unlimited
+ - ulimit -n 1024
+ - gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do
+ lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Windows
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+ windows_container:
+ dockerfile: ci/docker/windows_vs_2019
+ cpu: 4
+ memory: 4G
+ env:
+ PROVE_FLAGS: -j10
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+
+ sysinfo_script:
+ - chcp
+ - systeminfo
+ - powershell -Command get-psdrive -psprovider filesystem
+ - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+ - set
+
+ configure_script:
+ - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ # Disable file tracker, we're never going to rebuild...
+ - msbuild -m /p:TrackFileAccess=false pgsql.sln
+ tempinstall_script:
+ # Installation on windows currently only completely works from src\tools\msvc
+ - cd src\tools\msvc && perl .\install.pl %CIRRUS_WORKING_DIR%\tmp_install
+
+ check_test_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_test_script:
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+ plcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ isolationcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ modulescheck_test_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ contribcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ stop_test_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+ ssl_test_script:
+ - set with_ssl=openssl
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\ssl\
+ subscriptioncheck_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+ authentication_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+ recoverycheck_test_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ bincheck_test_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ upgradecheck_test_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ ecpgcheck_test_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src\tools\msvc
+ - perl vcregress.pl ecpgcheck
+
+ on_failure:
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: CompilerWarnings
+ depends_on:
+ - Linux
+ # task that did not run count as a success, so we need to recheck Linux' condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ container:
+ dockerfile: ci/docker/linux_debian_bullseye
+ env:
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ setup_script:
+ - echo "COPT=-Werror" > src/Makefile.custom
+ - gcc -v
+ - clang -v
+ # gcc with asserts disabled
+ always:
+ gcc_warning_script:
+ - ./configure --cache gcc.cache CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # gcc with asserts enabled
+ always:
+ gcc_a_warning_script:
+ - ./configure --cache gcc.cache --enable-cassert CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts disabled
+ always:
+ clang_warning_script:
+ - ./configure --cache clang.cache CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts enabled
+ always:
+ clang_a_warning_script:
+ - ./configure --cache clang.cache --enable-cassert CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..3fceab2e97b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+# Ignore everything, except ci/
+*
+!ci/*
diff --git a/ci/README b/ci/README
new file mode 100644
index 00000000000..566fd9fbd78
--- /dev/null
+++ b/ci/README
@@ -0,0 +1,36 @@
+Postgres Continuous Integration (CI)
+====================================
+
+Postgres has two forms of CI:
+
+1) All supported branches in the main postgres repository are continuously
+ tested via the the the buildfarm. As this covers only the main repository,
+ it cannot be used during development of features.
+
+ For details see https://buildfarm.postgresql.org/
+
+2) For not yet merged development work, CI can be enabled for some git hosting
+ providers. This allows to test patches on a number of platforms before they
+ are merged (or even submitted).
+
+
+Configuring CI on personal repositories
+=======================================
+
+Currently postgres contains CI support utilizing cirrus-ci. cirrus-ci
+currently is only available for github.
+
+
+Enabling cirrus-ci in a github repository
+=========================================
+
+To enable cirrus-ci on a repository, go to
+https://github.com/marketplace/cirrus-ci and select "Public
+Repositories". Then "Install it for free" and "Complete order". The next page
+allows to configure which repositories cirrus-ci has access too. Choose the
+relevant repository and "Install".
+
+Once enabled on a repository, future commits and pull-requests in that
+repository will automatically trigger CI builds. These are visible from the
+commit history / PRs, and can also be viewed in the cirrus-ci UI at
+https://cirrus-ci.com/github/<username>/<reponame>/
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..f6c1782f16b
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,13 @@
+FROM debian:bullseye
+RUN \
+ apt-get -y update && \
+ apt-get -y upgrade && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ git build-essential gcc g++ libreadline-dev flex bison make perl libipc-run-perl \
+ libio-pty-perl clang llvm-dev libperl-dev libpython3-dev tcl-dev libldap2-dev \
+ libicu-dev docbook-xml docbook-xsl fop libxml2-utils xsltproc krb5-admin-server \
+ krb5-kdc krb5-user slapd ldap-utils libssl-dev pkg-config locales-all liblz4-dev \
+ libsystemd-dev libxml2-dev libxslt1-dev python3-dev libkrb5-dev libpam-dev \
+ libkrb5-*-heimdal uuid-dev gettext \
+ liburing-dev python3-distutils ccache && \
+ apt-get clean
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..a4fcaceae96
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,111 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdk... ; `
+ echo 'starting sdk installation (for debugger)' ; `
+ Start-Process -Wait -FilePath ".\windsdksetup.exe" `
+ -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+ ; `
+ `
+ Start-Process -Wait -FilePath msiexec.exe `
+ -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+ ; `
+ C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+ `
+ Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+ Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install perl, python, flex and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's wy curl.exe is directly used here at least...
+#
+# Using perl 5.26.3.1 for now, as newer versions don't currently work correctly
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ echo 'adding to path, before setup below, so changes are not overwritten' ; `
+ setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o perl.zip `
+ https://strawberryperl.com/download/5.26.3.1/strawberry-perl-5.26.3.1-64bit-portable.zip ; `
+ echo 'installing perl' ; `
+ 7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+ `
+ curl.exe -sSL -o python.exe https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe ; `
+ echo 'installing python' ; `
+ Start-Process -Wait -FilePath ".\python.exe" `
+ -ArgumentList `
+ '/quiet', 'SimpleInstall=1', 'PrependPath=1', 'CompileAll=1', `
+ 'TargetDir=c:\python\', 'InstallAllUsers=1', 'Shortcuts=0', `
+ 'Include_docs=0', 'Include_tcltk=0', 'Include_tests=0' `
+ ; `
+ `
+ curl.exe -sSL -o winflexbison.zip `
+ https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+ echo 'installing winflexbison' ; `
+ 7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+ Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+ Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install openssl
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -o openssl-setup.exe -sSL https://slproweb.com/download/Win64OpenSSL-1_1_1L.exe ; `
+ echo 'staring openssl installation' ; `
+ Start-Process -Wait -FilePath ".\openssl-setup.exe" `
+ -ArgumentList '/DIR=c:\openssl\1.1.1l\ /VERYSILENT /SP- /SUPPRESSMSGBOXES' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install visual studio
+#
+# Adding VS path to vcvarsall.bat so user of container doesn't need to know the full path
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+ echo 'starting visual studio installation' ; `
+ Start-Process -Wait `
+ -FilePath c:\t\vs_buildtools.exe `
+ -ArgumentList `
+ '--quiet', '--wait', '--norestart', '--nocache', `
+ '--installPath', 'c:\BuildTools', `
+ '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+ '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+ Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\"
diff --git a/ci/freebsd_gcp_repartition.sh b/ci/freebsd_gcp_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/freebsd_gcp_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..637e3cfb343
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,12 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions=10
+
+# Settings that make logs more useful
+log_line_prefix='%m [%p][%b][%v:%x] '
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..ba82b13d69a
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,13 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+
+$config->{"openssl"} = "c:/openssl/1.1.1l/";
+$config->{"perl"} = "c:/strawberry/perl/";
+$config->{"python"} = "c:/python/";
+
+1;
--
2.23.0.385.gbc12974a89
--ep4fmvj5ysdv22gg
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v2-0002-windows-Improve-crash-assert-exception-handling.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v3 02/17] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
---
.cirrus.yml | 395 ++++++++++++++++++++++++++++++++
.dockerignore | 3 +
ci/docker/linux_debian_bullseye | 13 ++
ci/docker/windows_vs_2019 | 105 +++++++++
ci/freebsd_gcp_repartition.sh | 28 +++
ci/pg_ci_base.conf | 12 +
ci/windows_build_config.pl | 10 +
7 files changed, 566 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 .dockerignore
create mode 100644 ci/docker/linux_debian_bullseye
create mode 100644 ci/docker/windows_vs_2019
create mode 100755 ci/freebsd_gcp_repartition.sh
create mode 100644 ci/pg_ci_base.conf
create mode 100644 ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..f75bdce6dec
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,395 @@
+env:
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+ # target to test, for all but windows
+ CHECK: check-world
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ CCACHE_MAXSIZE: "500M"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ PG_TEST_EXTRA: kerberos ldap ssl
+
+
+task:
+ name: FreeBSD
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-freebsd-13-0
+ platform: freebsd
+ cpu: 2
+ memory: 2G
+ disk: 50
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ sysinfo_script:
+ - export || true
+ sysconfig_script:
+ - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+ repartition_script:
+ - ci/freebsd_gcp_repartition.sh
+ create_user_script:
+ - pw useradd postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
+ --with-uuid=bsd \
+ \
+ --with-includes=/usr/local/include --with-libs=/usr/local/lib \
+ CC="ccache cc"'
+ build_script:
+ - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - su postgres -c 'time gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Linux
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-bullseye
+ platform: linux
+ cpu: 4
+ memory: 2G
+ nested_virtualization: false
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - id
+ - uname -a
+ - cat /proc/cmdline
+ - lsblk
+ - ulimit -a -H
+ - ulimit -a -S
+ - export
+ sysconfig_script:
+ - useradd -m postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+ - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ - su postgres -c 'ulimit -l -H'
+ - su postgres -c 'ulimit -l -S'
+ - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-llvm \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-systemd \
+ --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" CFLAGS="-O0 -ggdb"'
+ build_script:
+ - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script: |
+ su postgres -c '\
+ ulimit -c unlimited; \
+ make -s ${CHECK} ${CHECKFLAGS} -j8 \
+ '
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: macOS
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+ osx_instance:
+ image: big-sur-base
+ env:
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ sysinfo_script:
+ - id
+ - export
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+ homebrew_cache:
+ folder: ${HOMEBREW_CACHE}
+ perl_cache:
+ folder: ~/perl5
+
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches:
+ - perl
+ core_install_script:
+ - sudo chmod 777 /cores
+ homebrew_install_script:
+ - brew install make coreutils ccache icu4c lz4 tcl-tk openldap
+ upload_caches:
+ - homebrew
+
+ configure_script: |
+ LIBS="/usr/local/lib:$LIBS"
+ INCLUDES="/usr/local/include:$INCLUDES"
+
+ INCLUDES="/usr/local/opt/openssl/include:$INCLUDES"
+ LIBS="/usr/local/opt/openssl/lib:$LIBS"
+
+ PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
+ INCLUDES="/usr/local/opt/icu4c/include:$INCLUDES"
+ LIBS="/usr/local/opt/icu4c/lib:$LIBS"
+
+ LIBS="/usr/local/opt/openldap/lib:$LIBS"
+ INCLUDES="/usr/local/opt/openldap/include:$INCLUDES"
+
+ export PKG_CONFIG_PATH
+
+ ./configure \
+ --prefix=$HOME/install \
+ --with-includes="$INCLUDES" \
+ --with-libs="$LIBS" \
+ \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/opt/tcl-tk/lib/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CFLAGS="-O0 -ggdb"
+ build_script:
+ - gmake -s -j12 && gmake -s -j12 -C contrib
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - ulimit -c unlimited
+ - ulimit -n 1024
+ - gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do
+ lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Windows
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+ windows_container:
+ dockerfile: ci/docker/windows_vs_2019
+ cpu: 4
+ memory: 4G
+ env:
+ PROVE_FLAGS: -j10
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+
+ sysinfo_script:
+ - chcp
+ - systeminfo
+ - powershell -Command get-psdrive -psprovider filesystem
+ - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+ - set
+
+ configure_script:
+ - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ # Disable file tracker, we're never going to rebuild...
+ - msbuild -m /p:TrackFileAccess=false pgsql.sln
+ tempinstall_script:
+ # Installation on windows currently only completely works from src\tools\msvc
+ - cd src\tools\msvc && perl .\install.pl %CIRRUS_WORKING_DIR%\tmp_install
+
+ check_test_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_test_script:
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+ plcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ isolationcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ modulescheck_test_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ contribcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ stop_test_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+ ssl_test_script:
+ - set with_ssl=openssl
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\ssl\
+ subscriptioncheck_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+ authentication_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+ recoverycheck_test_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ bincheck_test_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ upgradecheck_test_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ ecpgcheck_test_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src\tools\msvc
+ - perl vcregress.pl ecpgcheck
+
+ always:
+ cores_script:
+ - cat crashlog.txt || true
+ dump_artifacts:
+ path: "crashlog.txt"
+ type: text/plain
+
+ on_failure:
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: CompilerWarnings
+ depends_on:
+ - Linux
+ # task that did not run count as a success, so we need to recheck Linux' condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ container:
+ dockerfile: ci/docker/linux_debian_bullseye
+ env:
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ setup_script:
+ - echo "COPT=-Werror" > src/Makefile.custom
+ - gcc -v
+ - clang -v
+ # gcc with asserts disabled
+ always:
+ gcc_warning_script:
+ - ./configure --cache gcc.cache CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # gcc with asserts enabled
+ always:
+ gcc_a_warning_script:
+ - ./configure --cache gcc.cache --enable-cassert CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts disabled
+ always:
+ clang_warning_script:
+ - ./configure --cache clang.cache CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts enabled
+ always:
+ clang_a_warning_script:
+ - ./configure --cache clang.cache --enable-cassert CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..3fceab2e97b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+# Ignore everything, except ci/
+*
+!ci/*
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..f6c1782f16b
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,13 @@
+FROM debian:bullseye
+RUN \
+ apt-get -y update && \
+ apt-get -y upgrade && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ git build-essential gcc g++ libreadline-dev flex bison make perl libipc-run-perl \
+ libio-pty-perl clang llvm-dev libperl-dev libpython3-dev tcl-dev libldap2-dev \
+ libicu-dev docbook-xml docbook-xsl fop libxml2-utils xsltproc krb5-admin-server \
+ krb5-kdc krb5-user slapd ldap-utils libssl-dev pkg-config locales-all liblz4-dev \
+ libsystemd-dev libxml2-dev libxslt1-dev python3-dev libkrb5-dev libpam-dev \
+ libkrb5-*-heimdal uuid-dev gettext \
+ liburing-dev python3-distutils ccache && \
+ apt-get clean
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..e09ca0d5825
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,105 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+RUN `
+ New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio' ; `
+ New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' -Name KeepDownloadedPayloads -Value 0 -PropertyType DWord
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdk... ; `
+ Start-Process -Wait -FilePath ".\windsdksetup.exe" `
+ -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+ ; `
+ `
+ Start-Process -Wait -FilePath msiexec.exe `
+ -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+ ; `
+ C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t\* -Force -Recurse ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+ Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger; `
+ setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m
+
+
+# Install perl, python, flex and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's wy curl.exe is directly used here at least...
+#
+# Using perl 5.26.3.1 for now, as newer versions don't currently work correctly
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -sSL -o perl.zip `
+ https://strawberryperl.com/download/5.26.3.1/strawberry-perl-5.26.3.1-64bit-portable.zip ; `
+ 7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+ `
+ curl.exe -sSL -o python.exe https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe ; `
+ Start-Process -Wait -FilePath ".\python.exe" `
+ -ArgumentList `
+ '/quiet', 'SimpleInstall=1', 'PrependPath=1', 'CompileAll=1', `
+ 'TargetDir=c:\python\', 'InstallAllUsers=1', 'Shortcuts=0', `
+ 'Include_docs=0', 'Include_tcltk=0', 'Include_tests=0' `
+ ; `
+ `
+ curl.exe -sSL -o winflexbison.zip `
+ https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+ 7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+ Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+ Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m
+
+
+# Install visual studio
+#
+# Adding VS path to vcvarsall.bat so user of container doesn't need to know the full path
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+ Start-Process -Wait `
+ -FilePath c:\t\vs_buildtools.exe `
+ -ArgumentList `
+ '--quiet', '--wait', '--norestart', '--nocache', `
+ '--installPath', 'c:\BuildTools', `
+ '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+ '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348' ; `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+ Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\" ; `
+ setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m
+
+
+# Install openssl
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -o openssl-setup.exe -sSL https://slproweb.com/download/Win64OpenSSL-1_1_1L.exe ; `
+ Start-Process -Wait -FilePath ".\openssl-setup.exe" `
+ -ArgumentList '/DIR=c:\openssl\1.1.1l\ /VERYSILENT /SP- /SUPPRESSMSGBOXES' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
diff --git a/ci/freebsd_gcp_repartition.sh b/ci/freebsd_gcp_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/freebsd_gcp_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..637e3cfb343
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,12 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions=10
+
+# Settings that make logs more useful
+log_line_prefix='%m [%p][%b][%v:%x] '
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..bf0660416fa
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,10 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+$config->{"openssl"} = "c:/openssl/1.1.1l/";
+
+1;
--
2.23.0.385.gbc12974a89
--qozqt6hocm2wibt2
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v3-0003-fixup-ci-Add-CI-for-FreeBSD-Linux-MacOS-and-Windo.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v5 02/16] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci.
@ 2021-03-15 16:25 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2021-03-15 16:25 UTC (permalink / raw)
---
.cirrus.yml | 395 ++++++++++++++++++++++++++++++++
.dockerignore | 3 +
ci/docker/linux_debian_bullseye | 13 ++
ci/docker/windows_vs_2019 | 111 +++++++++
ci/freebsd_gcp_repartition.sh | 28 +++
ci/pg_ci_base.conf | 12 +
ci/windows_build_config.pl | 13 ++
7 files changed, 575 insertions(+)
create mode 100644 .cirrus.yml
create mode 100644 .dockerignore
create mode 100644 ci/docker/linux_debian_bullseye
create mode 100644 ci/docker/windows_vs_2019
create mode 100755 ci/freebsd_gcp_repartition.sh
create mode 100644 ci/pg_ci_base.conf
create mode 100644 ci/windows_build_config.pl
diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..f75bdce6dec
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,395 @@
+env:
+ # accelerate initial clone, but a bit of depth so that concurrent tasks work
+ CIRRUS_CLONE_DEPTH: 100
+ # Useful to be able to analyse what in a script takes long
+ CIRRUS_LOG_TIMESTAMP: true
+ # target to test, for all but windows
+ CHECK: check-world
+ CHECKFLAGS: -Otarget
+ PGCTLTIMEOUT: 120
+ CCACHE_MAXSIZE: "500M"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ PG_TEST_EXTRA: kerberos ldap ssl
+
+
+task:
+ name: FreeBSD
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-freebsd-13-0
+ platform: freebsd
+ cpu: 2
+ memory: 2G
+ disk: 50
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ sysinfo_script:
+ - export || true
+ sysconfig_script:
+ - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+ repartition_script:
+ - ci/freebsd_gcp_repartition.sh
+ create_user_script:
+ - pw useradd postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/lib/tcl8.6/ \
+ --with-uuid=bsd \
+ \
+ --with-includes=/usr/local/include --with-libs=/usr/local/lib \
+ CC="ccache cc"'
+ build_script:
+ - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - su postgres -c 'time gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Linux
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ compute_engine_instance:
+ image_project: pg-vm-images-aio
+ image: family/pg-aio-bullseye
+ platform: linux
+ cpu: 4
+ memory: 2G
+ nested_virtualization: false
+ env:
+ CCACHE_DIR: "/tmp/ccache_dir"
+ DEBUGINFOD_URLS: "https://debuginfod.debian.net";
+
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+
+ sysinfo_script:
+ - id
+ - uname -a
+ - cat /proc/cmdline
+ - lsblk
+ - ulimit -a -H
+ - ulimit -a -S
+ - export
+ sysconfig_script:
+ - useradd -m postgres
+ - chown -R postgres:postgres .
+ - mkdir -p /tmp/ccache_dir
+ - chown -R postgres:postgres /tmp/ccache_dir
+ - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+ - su postgres -c 'ulimit -l -H'
+ - su postgres -c 'ulimit -l -S'
+ - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+ configure_script: |
+ su postgres -c './configure \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-gssapi \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ --with-llvm \
+ --with-lz4 \
+ --with-pam \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-systemd \
+ --with-tcl --with-tclconfig=/usr/lib/tcl8.6/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang" CFLAGS="-O0 -ggdb"'
+ build_script:
+ - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+ upload_caches:
+ - ccache
+
+ tests_script: |
+ su postgres -c '\
+ ulimit -c unlimited; \
+ make -s ${CHECK} ${CHECKFLAGS} -j8 \
+ '
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do
+ binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ;
+ echo dumping $corefile for $binary ;
+ gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: macOS
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+ osx_instance:
+ image: big-sur-base
+ env:
+ CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ CCACHE_DIR: ${HOME}/ccache
+ HOMEBREW_CACHE: ${HOME}/homebrew-cache
+ PERL5LIB: ${HOME}/perl5/lib/perl5
+
+ sysinfo_script:
+ - id
+ - export
+ ccache_cache:
+ folder: ${CCACHE_DIR}
+ homebrew_cache:
+ folder: ${HOMEBREW_CACHE}
+ perl_cache:
+ folder: ~/perl5
+
+ cpan_install_script:
+ - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+ - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+ upload_caches:
+ - perl
+ core_install_script:
+ - sudo chmod 777 /cores
+ homebrew_install_script:
+ - brew install make coreutils ccache icu4c lz4 tcl-tk openldap
+ upload_caches:
+ - homebrew
+
+ configure_script: |
+ LIBS="/usr/local/lib:$LIBS"
+ INCLUDES="/usr/local/include:$INCLUDES"
+
+ INCLUDES="/usr/local/opt/openssl/include:$INCLUDES"
+ LIBS="/usr/local/opt/openssl/lib:$LIBS"
+
+ PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
+ INCLUDES="/usr/local/opt/icu4c/include:$INCLUDES"
+ LIBS="/usr/local/opt/icu4c/lib:$LIBS"
+
+ LIBS="/usr/local/opt/openldap/lib:$LIBS"
+ INCLUDES="/usr/local/opt/openldap/include:$INCLUDES"
+
+ export PKG_CONFIG_PATH
+
+ ./configure \
+ --prefix=$HOME/install \
+ --with-includes="$INCLUDES" \
+ --with-libs="$LIBS" \
+ \
+ --enable-cassert --enable-debug --enable-tap-tests \
+ --enable-nls \
+ \
+ --with-icu \
+ --with-ldap \
+ --with-libxml \
+ --with-libxslt \
+ \
+ --with-lz4 \
+ --with-perl \
+ --with-python \
+ --with-ssl=openssl \
+ --with-tcl --with-tclconfig=/usr/local/opt/tcl-tk/lib/ \
+ --with-uuid=e2fs \
+ \
+ CC="ccache gcc" CFLAGS="-O0 -ggdb"
+ build_script:
+ - gmake -s -j12 && gmake -s -j12 -C contrib
+ upload_caches:
+ - ccache
+
+ tests_script:
+ - ulimit -c unlimited
+ - ulimit -n 1024
+ - gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+ on_failure:
+ cores_script: |
+ for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do
+ lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ;
+ done
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: Windows
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+ windows_container:
+ dockerfile: ci/docker/windows_vs_2019
+ cpu: 4
+ memory: 4G
+ env:
+ PROVE_FLAGS: -j10
+ # The default working dir is in a directory msbuild complains about
+ CIRRUS_WORKING_DIR: "c:/cirrus"
+ TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+ # Avoid re-installing over and over
+ NO_TEMP_INSTALL: 1
+
+ sysinfo_script:
+ - chcp
+ - systeminfo
+ - powershell -Command get-psdrive -psprovider filesystem
+ - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+ - set
+
+ configure_script:
+ - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+ - vcvarsall x64
+ - perl src/tools/msvc/mkvcbuild.pl
+ build_script:
+ - vcvarsall x64
+ # Disable file tracker, we're never going to rebuild...
+ - msbuild -m /p:TrackFileAccess=false pgsql.sln
+ tempinstall_script:
+ # Installation on windows currently only completely works from src\tools\msvc
+ - cd src\tools\msvc && perl .\install.pl %CIRRUS_WORKING_DIR%\tmp_install
+
+ check_test_script:
+ - perl src/tools/msvc/vcregress.pl check parallel
+ startcreate_test_script:
+ - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+ - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+ - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+ plcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl plcheck
+ isolationcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl isolationcheck
+ modulescheck_test_script:
+ - perl src/tools/msvc/vcregress.pl modulescheck
+ contribcheck_test_script:
+ - perl src/tools/msvc/vcregress.pl contribcheck
+ stop_test_script:
+ - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+ ssl_test_script:
+ - set with_ssl=openssl
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\ssl\
+ subscriptioncheck_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+ authentication_test_script:
+ - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+ recoverycheck_test_script:
+ - perl src/tools/msvc/vcregress.pl recoverycheck
+ bincheck_test_script:
+ - perl src/tools/msvc/vcregress.pl bincheck
+ upgradecheck_test_script:
+ - perl src/tools/msvc/vcregress.pl upgradecheck
+ ecpgcheck_test_script:
+ # tries to build additional stuff
+ - vcvarsall x64
+ # References ecpg_regression.proj in the current dir
+ - cd src\tools\msvc
+ - perl vcregress.pl ecpgcheck
+
+ always:
+ cores_script:
+ - cat crashlog.txt || true
+ dump_artifacts:
+ path: "crashlog.txt"
+ type: text/plain
+
+ on_failure:
+ log_artifacts:
+ path: "**/**.log"
+ type: text/plain
+ regress_diffs_artifacts:
+ path: "**/**.diffs"
+ type: text/plain
+ tap_artifacts:
+ path: "**/regress_log_*"
+ type: text/plain
+
+
+task:
+ name: CompilerWarnings
+ depends_on:
+ - Linux
+ # task that did not run count as a success, so we need to recheck Linux' condition here :/
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+ container:
+ dockerfile: ci/docker/linux_debian_bullseye
+ env:
+ CCACHE_SIZE: "4GB"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ setup_script:
+ - echo "COPT=-Werror" > src/Makefile.custom
+ - gcc -v
+ - clang -v
+ # gcc with asserts disabled
+ always:
+ gcc_warning_script:
+ - ./configure --cache gcc.cache CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # gcc with asserts enabled
+ always:
+ gcc_a_warning_script:
+ - ./configure --cache gcc.cache --enable-cassert CC="ccache gcc"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts disabled
+ always:
+ clang_warning_script:
+ - ./configure --cache clang.cache CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
+ # clang with asserts enabled
+ always:
+ clang_a_warning_script:
+ - ./configure --cache clang.cache --enable-cassert CC="ccache clang"
+ - time make -s -j4 clean && time make -s -j4
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..3fceab2e97b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+# Ignore everything, except ci/
+*
+!ci/*
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..f6c1782f16b
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,13 @@
+FROM debian:bullseye
+RUN \
+ apt-get -y update && \
+ apt-get -y upgrade && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+ git build-essential gcc g++ libreadline-dev flex bison make perl libipc-run-perl \
+ libio-pty-perl clang llvm-dev libperl-dev libpython3-dev tcl-dev libldap2-dev \
+ libicu-dev docbook-xml docbook-xsl fop libxml2-utils xsltproc krb5-admin-server \
+ krb5-kdc krb5-user slapd ldap-utils libssl-dev pkg-config locales-all liblz4-dev \
+ libsystemd-dev libxml2-dev libxslt1-dev python3-dev libkrb5-dev libpam-dev \
+ libkrb5-*-heimdal uuid-dev gettext \
+ liburing-dev python3-distutils ccache && \
+ apt-get clean
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..a4fcaceae96
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,111 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdk... ; `
+ echo 'starting sdk installation (for debugger)' ; `
+ Start-Process -Wait -FilePath ".\windsdksetup.exe" `
+ -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+ ; `
+ `
+ Start-Process -Wait -FilePath msiexec.exe `
+ -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+ ; `
+ C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+ `
+ Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+ New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+ Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install perl, python, flex and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's wy curl.exe is directly used here at least...
+#
+# Using perl 5.26.3.1 for now, as newer versions don't currently work correctly
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ echo 'adding to path, before setup below, so changes are not overwritten' ; `
+ setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o perl.zip `
+ https://strawberryperl.com/download/5.26.3.1/strawberry-perl-5.26.3.1-64bit-portable.zip ; `
+ echo 'installing perl' ; `
+ 7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+ `
+ curl.exe -sSL -o python.exe https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe ; `
+ echo 'installing python' ; `
+ Start-Process -Wait -FilePath ".\python.exe" `
+ -ArgumentList `
+ '/quiet', 'SimpleInstall=1', 'PrependPath=1', 'CompileAll=1', `
+ 'TargetDir=c:\python\', 'InstallAllUsers=1', 'Shortcuts=0', `
+ 'Include_docs=0', 'Include_tcltk=0', 'Include_tests=0' `
+ ; `
+ `
+ curl.exe -sSL -o winflexbison.zip `
+ https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+ echo 'installing winflexbison' ; `
+ 7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+ Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+ Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install openssl
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ `
+ curl.exe -o openssl-setup.exe -sSL https://slproweb.com/download/Win64OpenSSL-1_1_1L.exe ; `
+ echo 'staring openssl installation' ; `
+ Start-Process -Wait -FilePath ".\openssl-setup.exe" `
+ -ArgumentList '/DIR=c:\openssl\1.1.1l\ /VERYSILENT /SP- /SUPPRESSMSGBOXES' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse
+
+
+# Install visual studio
+#
+# Adding VS path to vcvarsall.bat so user of container doesn't need to know the full path
+RUN `
+ mkdir c:\t ; `
+ cd c:\t ; `
+ setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m ; `
+ `
+ curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+ echo 'starting visual studio installation' ; `
+ Start-Process -Wait `
+ -FilePath c:\t\vs_buildtools.exe `
+ -ArgumentList `
+ '--quiet', '--wait', '--norestart', '--nocache', `
+ '--installPath', 'c:\BuildTools', `
+ '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+ '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348' ; `
+ `
+ cd c:\ ; `
+ Remove-Item C:\t -Force -Recurse ; `
+ Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+ Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\"
diff --git a/ci/freebsd_gcp_repartition.sh b/ci/freebsd_gcp_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/freebsd_gcp_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..637e3cfb343
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,12 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions=10
+
+# Settings that make logs more useful
+log_line_prefix='%m [%p][%b][%v:%x] '
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..ba82b13d69a
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,13 @@
+use strict;
+use warnings;
+
+our $config;
+
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+
+$config->{"openssl"} = "c:/openssl/1.1.1l/";
+$config->{"perl"} = "c:/strawberry/perl/";
+$config->{"python"} = "c:/python/";
+
+1;
--
2.23.0.385.gbc12974a89
--q7iu5jwpuq7sbl3z
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v5-0003-plpython-Drop-support-python2.patch"
^ permalink raw reply [nested|flat] 7+ messages in thread
* =?gb2312?B?u9i4tDogV0FMIEluc2VydGlvbiBMb2NrIEltcHJvdmVtZW50cw==?=
@ 2023-03-23 02:51 adherent postgres <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: adherent postgres @ 2023-03-23 02:51 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Andres Freund <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi Andres Freund
This patch improves performance significantly,Commitfest 2023-03 is coming to an end,Is it not submitted yet since the patch still needs to be improved?
Best wish
________________________________
发件人: Nathan Bossart <[email protected]>
发送时间: 2023年2月21日 13:49
收件人: Bharath Rupireddy <[email protected]>
抄送: Andres Freund <[email protected]>; PostgreSQL Hackers <[email protected]>
主题: Re: WAL Insertion Lock Improvements
On Thu, Feb 09, 2023 at 11:51:28AM +0530, Bharath Rupireddy wrote:
> On Thu, Feb 9, 2023 at 3:36 AM Nathan Bossart <[email protected]> wrote:
>> Overall, I think this patch is in reasonable shape.
>
> Thanks for reviewing. Please see the attached v5 patch.
I'm marking this as ready-for-committer. I think a couple of the comments
could use some small adjustments, but that probably doesn't need to hold up
this patch.
--
Nathan Bossart
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2023-03-23 02:51 UTC | newest]
Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-15 16:25 [PATCH v1 2/6] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2021-03-15 16:25 [PATCH v3 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2021-03-15 16:25 [PATCH v5 02/16] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2021-03-15 16:25 [PATCH v4 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2021-03-15 16:25 [PATCH v3 02/17] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2021-03-15 16:25 [PATCH v2 1/2] ci: Add CI for FreeBSD, Linux, MacOS and Windows, utilizing cirrus-ci. Andres Freund <[email protected]>
2023-03-23 02:51 =?gb2312?B?u9i4tDogV0FMIEluc2VydGlvbiBMb2NrIEltcHJvdmVtZW50cw==?= adherent postgres <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox