From 4e20afb9677fe443f274b7e91ee99bfab874003d Mon Sep 17 00:00:00 2001 From: Jacob Champion Date: Wed, 13 Aug 2025 10:58:56 -0700 Subject: [PATCH v4 2/7] Add support for pytest test suites Specify --enable-pytest/-Dpytest=enabled at configure time. This contains no Postgres test logic -- it is just a "vanilla" pytest skeleton. I've written a custom pgtap output plugin, used by the Meson mtest runner, to fully control what we see during CI test failures. The pytest-tap plugin would have been preferable, but it's now in maintenance mode, and it has problems with accidentally suppressing important collection failures. test_something.py is intended to show a sample failure in the CI. TODOs: - OpenBSD has an ANSI-related terminal bug, but I'm not sure if the bug is in Cirrus, the image, pytest, Python, or readline. The TERM envvar is unset to work around it. If this workaround is removed, a bad ANSI escape is inserted into the pgtap output and mtest is unable to parse it. - The Chocolatey CI setup is subpar. Need to find a way to bless the dependencies in use rather than pulling from pip... or maybe that will be done by the image baker. --- .cirrus.tasks.yml | 38 ++++-- .gitignore | 1 + config/check_pytest.py | 150 ++++++++++++++++++++++++ config/conftest.py | 18 +++ config/pytest-requirements.txt | 21 ++++ configure | 108 ++++++++++++++++- configure.ac | 25 +++- meson.build | 92 +++++++++++++++ meson_options.txt | 8 +- pytest.ini | 6 + src/Makefile.global.in | 23 ++++ src/makefiles/meson.build | 2 + src/test/Makefile | 11 +- src/test/meson.build | 1 + src/test/pytest/Makefile | 20 ++++ src/test/pytest/README | 1 + src/test/pytest/meson.build | 16 +++ src/test/pytest/plugins/pgtap.py | 192 +++++++++++++++++++++++++++++++ 18 files changed, 718 insertions(+), 15 deletions(-) create mode 100644 config/check_pytest.py create mode 100644 config/conftest.py create mode 100644 config/pytest-requirements.txt create mode 100644 pytest.ini create mode 100644 src/test/pytest/Makefile create mode 100644 src/test/pytest/README create mode 100644 src/test/pytest/meson.build create mode 100644 src/test/pytest/plugins/pgtap.py diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 038d043d00e..ee2084bdfb6 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -21,7 +21,8 @@ env: # target to test, for all but windows CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS - CHECKFLAGS: -Otarget + # TODO were we avoiding --keep-going on purpose? + CHECKFLAGS: -Otarget --keep-going PROVE_FLAGS: --timer # Build test dependencies as part of the build step, to see compiler # errors/warnings in one place. @@ -44,6 +45,7 @@ env: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled -Dplperl=enabled -Dplpython=enabled -Ddocs=enabled @@ -225,7 +227,9 @@ task: chown root:postgres /tmp/cores sysctl kern.corefile='/tmp/cores/%N.%P.core' setup_additional_packages_script: | - #pkg install -y ... + pkg install -y \ + py311-packaging \ + py311-pytest # NB: Intentionally build without -Dllvm. The freebsd image size is already # large enough to make VM startup slow, and even without llvm freebsd @@ -317,7 +321,10 @@ task: -Dpam=enabled setup_additional_packages_script: | - #pkgin -y install ... + pkgin -y install \ + py312-packaging \ + py312-test + ln -s /usr/pkg/bin/pytest-3.12 /usr/pkg/bin/pytest <<: *netbsd_task_template - name: OpenBSD - Meson @@ -329,6 +336,7 @@ task: IMAGE_FAMILY: pg-ci-openbsd-postgres PKGCONFIG_PATH: '/usr/lib/pkgconfig:/usr/local/lib/pkgconfig' CORE_DUMP_EXECUTABLE_DIR: $CIRRUS_WORKING_DIR/build/tmp_install/usr/local/pgsql/bin + TERM: # TODO why does pytest print ANSI escapes on OpenBSD? MESON_FEATURES: >- -Dbsd_auth=enabled @@ -337,7 +345,9 @@ task: -Duuid=e2fs setup_additional_packages_script: | - #pkg_add -I ... + pkg_add -I \ + py3-test \ + py3-packaging # Always core dump to ${CORE_DUMP_DIR} set_core_dump_script: sysctl -w kern.nosuidcoredump=2 <<: *openbsd_task_template @@ -496,8 +506,10 @@ task: EOF setup_additional_packages_script: | - #apt-get update - #DEBIAN_FRONTEND=noninteractive apt-get -y install ... + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + python3-pytest \ + python3-packaging matrix: # SPECIAL: @@ -521,14 +533,15 @@ task: set -e ./configure \ --enable-cassert --enable-injection-points --enable-debug \ - --enable-tap-tests --enable-nls \ + --enable-tap-tests --enable-pytest --enable-nls \ --with-segsize-blocks=6 \ --with-libnuma \ --with-liburing \ \ ${LINUX_CONFIGURE_FEATURES} \ \ - CLANG="ccache clang" + CLANG="ccache clang" \ + PYTEST="env LD_PRELOAD=/lib/x86_64-linux-gnu/libasan.so.8 pytest" EOF build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin" upload_caches: ccache @@ -665,6 +678,8 @@ task: p5.34-io-tty p5.34-ipc-run python312 + py312-packaging + py312-pytest tcl zstd @@ -714,6 +729,7 @@ task: sh src/tools/ci/ci_macports_packages.sh $MACOS_PACKAGE_LIST # system python doesn't provide headers sudo /opt/local/bin/port select python3 python312 + sudo /opt/local/bin/port select pytest pytest312 # Make macports install visible for subsequent steps echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV upload_caches: macports @@ -787,6 +803,8 @@ task: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled + -DPYTEST=c:\Windows\system32\config\systemprofile\AppData\Roaming\Python\Python310\Scripts\pytest.exe -Dplperl=enabled -Dplpython=enabled @@ -795,8 +813,10 @@ task: depends_on: SanityCheck only_if: $CI_WINDOWS_ENABLED + # XXX Does Chocolatey really not have any Python package installers? setup_additional_packages_script: | REM choco install -y --no-progress ... + pip3 install --user packaging pytest setup_hosts_file_script: | echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts @@ -859,7 +879,7 @@ task: folder: ${CCACHE_DIR} setup_additional_packages_script: | - REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ... + C:\msys64\usr\bin\pacman.exe -S --noconfirm mingw-w64-ucrt-x86_64-python-packaging mingw-w64-ucrt-x86_64-python-pytest mingw_info_script: | %BASH% -c "where gcc" diff --git a/.gitignore b/.gitignore index 4e911395fe3..268426003b1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ win32ver.rc *.exe lib*dll.def lib*.pc +__pycache__/ # Local excludes in root directory /GNUmakefile diff --git a/config/check_pytest.py b/config/check_pytest.py new file mode 100644 index 00000000000..1562d16bcda --- /dev/null +++ b/config/check_pytest.py @@ -0,0 +1,150 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group +# +# Verify that pytest-requirements.txt is satisfied. This would probably be +# easier with pip, but requiring pip on build machines is a non-starter for +# many. +# +# This is coded as a pytest suite in order to check the Python distribution in +# use by pytest, as opposed to the Python distribution being linked against +# Postgres. In some setups they are separate. +# +# The design philosophy of this script is to bend over backwards to help people +# figure out what is missing. The target audience for error output is the +# buildfarm operator who just wants to get the tests running, not the test +# developer who presumably already knows how to solve these problems. + +import importlib +import sys +from typing import List, Union # needed for earlier Python versions + +# importlib.metadata is part of the standard library from 3.8 onwards. Earlier +# Python versions have an official backport called importlib_metadata, which can +# generally be installed as a separate OS package (python3-importlib-metadata). +# This complication can be removed once we stop supporting Python 3.7. +try: + from importlib import metadata +except ImportError: + try: + import importlib_metadata as metadata + except ImportError: + # package_version() will need to fall back. This is unlikely to happen + # in practice, because pytest 7.x depends on importlib_metadata itself. + metadata = None + + +def report(*args): + """ + Prints a configure-time message to the user. (The configure scripts will + display these messages and ignore the output from the pytest suite.) This + assumes --capture=no is in use, to avoid pytest's standard stream capture. + """ + print(*args, file=sys.stderr) + + +def package_version(pkg: str) -> Union[str, None]: + """ + Returns the version of the named package, or None if the package is not + installed. + + This function prefers to use the distribution package version, if we have + the necessary prerequisites. Otherwise it will fall back to the __version__ + of the imported module, which aligns with pytest.importorskip(). + """ + if metadata is not None: + try: + return metadata.version(pkg) + except metadata.PackageNotFoundError: + return None + + # This is an older Python and we don't have importlib_metadata. Fall back to + # __version__ instead. + try: + mod = importlib.import_module(pkg) + except ModuleNotFoundError: + return None + + if hasattr(mod, "__version__"): + return mod.__version__ + + # We're out of options. If this turns out to cause problems in practice, we + # might need to require importlib_metadata on older buildfarm members. But + # since our top-level requirements list will be small, and this possibility + # will eventually age out with newer Pythons, don't spend more effort on + # this case for now. + report(f"Fix check_pytest.py! {pkg} has no __version__") + assert False, "internal error in package_version()" + + +def packaging_check(requirements: List[str]) -> bool: + """ + Reports the status of each required package to the configure program. + Returns True if all dependencies were found. + """ + report() # an opening newline makes the configure output easier to read + + try: + # packaging contains the PyPA definitions of requirement specifiers. + # This is contained in a separate OS package (for example, + # python3-packaging), but it's extremely likely that the user has it + # installed already, because modern versions of pytest depend on it too. + import packaging + from packaging.requirements import Requirement + + except ImportError as err: + # We don't even have enough prerequisites to check our prerequisites. + # Print the import error as-is. + report(err) + return False + + # Strip extraneous whitespace, whole-line comments, and empty lines from our + # specifier list. + requirements = [r.strip() for r in requirements] + requirements = [r for r in requirements if r and r[0] != "#"] + + found = True + for spec in requirements: + req = Requirement(spec) + + # Skip any packages marked as unneeded for this particular Python env. + if req.marker and not req.marker.evaluate(): + continue + + # Make sure the package is installed... + version = package_version(req.name) + if version is None: + report(f"package '{req.name}': not installed") + found = False + continue + + # ...and that it has a compatible version. + if not req.specifier.contains(version): + report( + "package '{}': has version {}, but '{}' is required".format( + req.name, version, req.specifier + ), + ) + found = False + continue + + # Report installed packages too, to mirror check_modules.pl. + report(f"package '{req.name}': installed (version {version})") + + return found + + +def test_packages(requirements_file): + """ + Entry point. + """ + try: + with open(requirements_file, "r") as f: + requirements = f.readlines() + + all_found = packaging_check(requirements) + + except Exception as err: + # Surface any breakage to the configure script before failing the test. + report(err) + raise + + assert all_found, "required packages are missing" diff --git a/config/conftest.py b/config/conftest.py new file mode 100644 index 00000000000..a9c2bc546e8 --- /dev/null +++ b/config/conftest.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group +# +# Support for check_pytest.py. The configure script provides the path to +# pytest-requirements.txt via the --requirements option added here. + +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--requirements", + help="path to pytest-requirements.txt", + ) + + +@pytest.fixture +def requirements_file(request): + return request.config.getoption("--requirements") diff --git a/config/pytest-requirements.txt b/config/pytest-requirements.txt new file mode 100644 index 00000000000..1c5e283d1e2 --- /dev/null +++ b/config/pytest-requirements.txt @@ -0,0 +1,21 @@ +# +# This file contains the Python packages which are required in order for us to +# enable pytest. +# +# The syntax is a *subset* of pip's requirements.txt syntax, so that both pip +# and check_pytest.py can use it. Only whole-line comments and standard Python +# dependency specifiers are allowed. pip-specific goodies like includes and +# environment substitutions are not supported; keep it simple. +# +# Packages belong here if their absence should cause a configuration failure. If +# you'd like to make a package optional, consider using pytest.importorskip() +# instead. +# + +# pytest 7.0 was the last version which supported Python 3.6, but the BSDs have +# started putting 8.x into ports, so we support both. (pytest 8 can be used +# throughout once we drop support for Python 3.7.) +pytest >= 7.0, < 10 + +# packaging is used by check_pytest.py at configure time. +packaging diff --git a/configure b/configure index 14ad0a5006f..d6fe7d3d293 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ vpath_build PG_SYSROOT PG_VERSION_NUM LDFLAGS_EX_BE +PYTEST PROVE DBTOEPUB FOP @@ -772,6 +773,7 @@ CFLAGS CC enable_injection_points PG_TEST_EXTRA +enable_pytest enable_tap_tests enable_dtrace DTRACEFLAGS @@ -850,6 +852,7 @@ enable_profiling enable_coverage enable_dtrace enable_tap_tests +enable_pytest enable_injection_points with_blocksize with_segsize @@ -1550,7 +1553,10 @@ Optional Features: --enable-profiling build with profiling enabled --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support - --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) + --enable-tap-tests enable (Perl-based) TAP tests (requires Perl and + IPC::Run) + --enable-pytest enable (Python-based) pytest suites (requires + Python) --enable-injection-points enable injection points (for testing) --enable-depend turn on automatic dependency tracking @@ -3632,7 +3638,7 @@ fi # -# TAP tests +# Test frameworks # @@ -3660,6 +3666,32 @@ fi + +# Check whether --enable-pytest was given. +if test "${enable_pytest+set}" = set; then : + enableval=$enable_pytest; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-pytest option" "$LINENO" 5 + ;; + esac + +else + enable_pytest=no + +fi + + + + + + # # Injection points # @@ -19229,6 +19261,78 @@ $as_echo "$modulestderr" >&6; } fi fi +if test "$enable_pytest" = yes; then + if test -z "$PYTEST"; then + for ac_prog in pytest py.test +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTEST+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTEST in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTEST="$PYTEST" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTEST="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTEST=$ac_cv_path_PYTEST +if test -n "$PYTEST"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTEST" && break +done + +else + # Report the value of PYTEST in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTEST" >&5 +$as_echo_n "checking for PYTEST... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +fi + + if test -z "$PYTEST"; then + as_fn_error $? "pytest not found" "$LINENO" 5 + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python packages required for pytest" >&5 +$as_echo_n "checking for Python packages required for pytest... " >&6; } + modulestderr=`$PYTEST -c "$srcdir/pytest.ini" --confcutdir="$srcdir/config" --capture=no "$srcdir/config/check_pytest.py" --requirements "$srcdir/config/pytest-requirements.txt" 2>&1 >/dev/null` + if test $? -eq 0; then + echo "$modulestderr" >&5 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $modulestderr" >&5 +$as_echo "$modulestderr" >&6; } + as_fn_error $? "Additional Python packages are required to run the pytest suites" "$LINENO" 5 + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/configure.ac b/configure.ac index 01b3bbc1be8..d513d374f3e 100644 --- a/configure.ac +++ b/configure.ac @@ -225,11 +225,16 @@ AC_SUBST(DTRACEFLAGS)]) AC_SUBST(enable_dtrace) # -# TAP tests +# Test frameworks # PGAC_ARG_BOOL(enable, tap-tests, no, - [enable TAP tests (requires Perl and IPC::Run)]) + [enable (Perl-based) TAP tests (requires Perl and IPC::Run)]) AC_SUBST(enable_tap_tests) + +PGAC_ARG_BOOL(enable, pytest, no, + [enable (Python-based) pytest suites (requires Python)]) +AC_SUBST(enable_pytest) + AC_ARG_VAR(PG_TEST_EXTRA, [enable selected extra tests (overridden at runtime by PG_TEST_EXTRA environment variable)]) @@ -2412,6 +2417,22 @@ if test "$enable_tap_tests" = yes; then fi fi +if test "$enable_pytest" = yes; then + PGAC_PATH_PROGS(PYTEST, pytest py.test) + if test -z "$PYTEST"; then + AC_MSG_ERROR([pytest not found]) + fi + AC_MSG_CHECKING(for Python packages required for pytest) + [modulestderr=`$PYTEST -c "$srcdir/pytest.ini" --confcutdir="$srcdir/config" --capture=no "$srcdir/config/check_pytest.py" --requirements "$srcdir/config/pytest-requirements.txt" 2>&1 >/dev/null`] + if test $? -eq 0; then + echo "$modulestderr" >&AS_MESSAGE_LOG_FD + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT([$modulestderr]) + AC_MSG_ERROR([Additional Python packages are required to run the pytest suites]) + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/meson.build b/meson.build index 551e27f5eb3..e27c8ad4455 100644 --- a/meson.build +++ b/meson.build @@ -1711,6 +1711,39 @@ endif +############################################################### +# Library: pytest +############################################################### + +pytest_enabled = false +pytest = not_found_dep + +pytestopt = get_option('pytest') +if not pytestopt.disabled() + pytest = find_program(get_option('PYTEST'), native: true, required: pytestopt) + if pytest.found() + pytest_check = run_command(pytest, + '-c', 'pytest.ini', + '--confcutdir=config', + '--capture=no', + 'config/check_pytest.py', + '--requirements', 'config/pytest-requirements.txt', + check: false) + if pytest_check.returncode() != 0 + message(pytest_check.stderr()) + if pytestopt.enabled() + error('Additional Python packages are required to run the pytest suites.') + else + warning('Additional Python packages are required to run the pytest suites.') + endif + else + pytest_enabled = true + endif + endif +endif + + + ############################################################### # Library: zstd ############################################################### @@ -3808,6 +3841,63 @@ foreach test_dir : tests ) endforeach install_suites += test_group + elif kind == 'pytest' + testwrap_pytest = testwrap_base + if not pytest_enabled + testwrap_pytest += ['--skip', 'pytest not enabled'] + endif + + test_command = [ + pytest.full_path(), + '-c', meson.project_source_root() / 'pytest.ini', + '--verbose', + '-p', 'pgtap', # enable our test reporter plugin + '-ra', # show skipped and xfailed tests too + ] + + # Add temporary install, the build directory for non-installed binaries and + # also test/ for non-installed test binaries built separately. + env = test_env + env.prepend('PATH', temp_install_bindir, test_dir['bd'], test_dir['bd'] / 'test') + temp_install_datadir = '@0@@1@'.format(test_install_destdir, dir_prefix / dir_data) + env.set('share_contrib_dir', temp_install_datadir / 'contrib') + env.prepend('PYTHONPATH', meson.project_source_root() / 'src' / 'test' / 'pytest' / 'plugins') + + foreach name, value : t.get('env', {}) + env.set(name, value) + endforeach + + test_group = test_dir['name'] + test_kwargs = { + 'protocol': 'tap', + 'suite': test_group, + 'timeout': 1000, + 'depends': test_deps + t.get('deps', []), + 'env': env, + } + t.get('test_kwargs', {}) + + foreach onetest : t['tests'] + # Make test names prettier, remove pyt/ and .py + onetest_p = onetest + if onetest_p.startswith('pyt/') + onetest_p = onetest.split('pyt/')[1] + endif + if onetest_p.endswith('.py') + onetest_p = fs.stem(onetest_p) + endif + + test(test_dir['name'] / onetest_p, + python, + kwargs: test_kwargs, + args: testwrap_pytest + [ + '--testgroup', test_dir['name'], + '--testname', onetest_p, + '--', test_command, + test_dir['sd'] / onetest, + ], + ) + endforeach + install_suites += test_group else error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd'])) endif @@ -3982,6 +4072,7 @@ summary( 'dtrace': dtrace, 'flex': '@0@ @1@'.format(flex.full_path(), flex_version), 'prove': prove, + 'pytest': pytest, }, section: 'Programs', ) @@ -4022,6 +4113,7 @@ summary( summary( { 'tap': tap_tests_enabled, + 'pytest': pytest_enabled, }, section: 'Other features', list_sep: ' ', diff --git a/meson_options.txt b/meson_options.txt index 06bf5627d3c..88f22e699d9 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -41,7 +41,10 @@ option('cassert', type: 'boolean', value: false, description: 'Enable assertion checks (for debugging)') option('tap_tests', type: 'feature', value: 'auto', - description: 'Enable TAP tests') + description: 'Enable (Perl-based) TAP tests') + +option('pytest', type: 'feature', value: 'auto', + description: 'Enable (Python-based) pytest suites') option('injection_points', type: 'boolean', value: false, description: 'Enable injection points') @@ -195,6 +198,9 @@ option('PERL', type: 'string', value: 'perl', option('PROVE', type: 'string', value: 'prove', description: 'Path to prove binary') +option('PYTEST', type: 'array', value: ['pytest', 'py.test'], + description: 'Path to pytest binary') + option('PYTHON', type: 'array', value: ['python3', 'python'], description: 'Path to python binary') diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..8e8388f3afc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +minversion = 7.0 + +# Ignore ./config (which contains the configure-time check_pytest.py tests) by +# default. +addopts = --ignore ./config diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 371cd7eba2c..39e67358289 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -211,6 +211,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_pytest = @enable_pytest@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ @@ -354,6 +355,7 @@ MSGFMT = @MSGFMT@ MSGFMT_FLAGS = @MSGFMT_FLAGS@ MSGMERGE = @MSGMERGE@ OPENSSL = @OPENSSL@ +PYTEST = @PYTEST@ PYTHON = @PYTHON@ TAR = @TAR@ XGETTEXT = @XGETTEXT@ @@ -508,6 +510,27 @@ prove_installcheck = @echo "TAP tests not enabled. Try configuring with --enable prove_check = $(prove_installcheck) endif +ifeq ($(enable_pytest),yes) + +pytest_installcheck = @echo "Installcheck is not currently supported for pytest." + +define pytest_check +echo "# +++ pytest check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest/plugins:$$PYTHONPATH' \ + $(with_temp_install) \ + $(PYTEST) -c '$(abs_top_srcdir)/pytest.ini' --verbose -ra ./pyt/ +endef + +else +pytest_installcheck = @echo "pytest is not enabled. Try configuring with --enable-pytest" +pytest_check = $(pytest_installcheck) +endif + # Installation. install_bin = @install_bin@ diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index c6edf14ec44..5b9a804aa94 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -56,6 +56,7 @@ pgxs_kv = { 'enable_nls': libintl.found() ? 'yes' : 'no', 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', 'enable_tap_tests': tap_tests_enabled ? 'yes' : 'no', + 'enable_pytest': pytest_enabled ? 'yes' : 'no', 'enable_debug': get_option('debug') ? 'yes' : 'no', 'enable_coverage': 'no', 'enable_dtrace': dtrace.found() ? 'yes' : 'no', @@ -145,6 +146,7 @@ pgxs_bins = { 'OPENSSL': openssl, 'PERL': perl, 'PROVE': prove, + 'PYTEST': pytest, 'PYTHON': python, 'TAR': tar, 'ZSTD': program_zstd, diff --git a/src/test/Makefile b/src/test/Makefile index 511a72e6238..0be9771d71f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,16 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription +SUBDIRS = \ + authentication \ + isolation \ + modules \ + perl \ + postmaster \ + pytest \ + recovery \ + regress \ + subscription ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/meson.build b/src/test/meson.build index ccc31d6a86a..d08a6ef61c2 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -5,6 +5,7 @@ subdir('isolation') subdir('authentication') subdir('postmaster') +subdir('pytest') subdir('recovery') subdir('subscription') subdir('modules') diff --git a/src/test/pytest/Makefile b/src/test/pytest/Makefile new file mode 100644 index 00000000000..2bdca96ccbe --- /dev/null +++ b/src/test/pytest/Makefile @@ -0,0 +1,20 @@ +#------------------------------------------------------------------------- +# +# Makefile for pytest +# +# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/pytest/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/pytest +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(pytest_check) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/pytest/README b/src/test/pytest/README new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/src/test/pytest/README @@ -0,0 +1 @@ +TODO diff --git a/src/test/pytest/meson.build b/src/test/pytest/meson.build new file mode 100644 index 00000000000..abd128dfa24 --- /dev/null +++ b/src/test/pytest/meson.build @@ -0,0 +1,16 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +if not pytest_enabled + subdir_done() +endif + +tests += { + 'name': 'pytest', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'pytest': { + 'tests': [ + 'pyt/test_something.py', + ], + }, +} diff --git a/src/test/pytest/plugins/pgtap.py b/src/test/pytest/plugins/pgtap.py new file mode 100644 index 00000000000..6a729d252e1 --- /dev/null +++ b/src/test/pytest/plugins/pgtap.py @@ -0,0 +1,192 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +import os +import sys + +import pytest + +# +# Helpers +# + + +class TAP: + """ + A basic API for reporting via the TAP protocol. + """ + + def __init__(self): + self.count = 0 + + # XXX interacts poorly with testwrap's boilerplate diagnostics + # self.print("TAP version 13") + + def expect(self, num: int): + self.print(f"1..{num}") + + def print(self, *args): + print(*args, file=sys.__stdout__) + + def ok(self, name: str): + self.count += 1 + self.print("ok", self.count, "-", name) + + def skip(self, name: str, reason: str): + self.count += 1 + self.print("ok", self.count, "-", name, "# skip", reason) + + def fail(self, name: str, details: str): + self.count += 1 + self.print("not ok", self.count, "-", name) + + # mtest has some odd behavior around TAP tests where it won't print + # diagnostics on failure if they're part of the stdout stream, so we + # might as well just dump the details directly to stderr instead. + print(details, file=sys.__stderr__) + + +tap = TAP() + + +class TestNotes: + """ + Annotations for a single test. The existing pytest hooks keep interesting + information somewhat separated across the different stages + (setup/test/teardown), so this class is used to correlate them. + """ + + skipped = False + skip_reason = None + + failed = False + details = "" + + +# Register a custom key in the stash dictionary for keeping our TestNotes. +notes_key = pytest.StashKey[TestNotes]() + + +# +# Hook Implementations +# + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + """ + Hijacks the standard streams as soon as possible during pytest startup. The + pytest-formatted output gets logged to file instead, and we'll use the + original sys.__stdout__/__stderr__ streams for the TAP protocol. + """ + logdir = os.getenv("TESTLOGDIR") + if not logdir: + raise RuntimeError("pgtap requires the TESTLOGDIR envvar to be set") + + os.makedirs(logdir) + logpath = os.path.join(logdir, "pytest.log") + sys.stdout = sys.stderr = open(logpath, "a", buffering=1) + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session, exitstatus): + """ + Suppresses nonzero exit codes due to failed tests. (In that case, we want + Meson to report a failure count, not a generic ERROR.) + """ + if exitstatus == pytest.ExitCode.TESTS_FAILED: + session.exitstatus = pytest.ExitCode.OK + + +@pytest.hookimpl +def pytest_collectreport(report): + # Include collection failures directly in Meson error output. + if report.failed: + print(report.longreprtext, file=sys.__stderr__) + + +@pytest.hookimpl +def pytest_internalerror(excrepr, excinfo): + # Include internal errors directly in Meson error output. + print(excrepr, file=sys.__stderr__) + + +# +# Hook Wrappers +# +# In pytest parlance, a "wrapper" for a hook can inspect and optionally modify +# existing hooks' behavior, but it does not replace the hook chain. This is done +# through a generator-style API which chains the hooks together (see the use of +# `yield`). +# + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collection(session): + """Reports the number of gathered tests after collection is finished.""" + res = yield + tap.expect(session.testscollected) + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Annotates a test item with our TestNotes and grabs relevant information for + reporting. + + This is called multiple times per test, so it's not correct to print the TAP + result here. (A test and its teardown stage can both fail, and we want to + see the details for both.) We instead combine all the information for use by + our pytest_runtest_protocol wrapper later on. + """ + res = yield + + if notes_key not in item.stash: + item.stash[notes_key] = TestNotes() + notes = item.stash[notes_key] + + report = res.get_result() + if report.passed: + pass # no annotation needed + + elif report.skipped: + notes.skipped = True + _, _, notes.skip_reason = report.longrepr + + elif report.failed: + notes.failed = True + + if not notes.details: + notes.details += "{:_^72}\n\n".format(f" {report.head_line} ") + + if report.when in ("setup", "teardown"): + notes.details += "\n{:_^72}\n\n".format( + f" Error during {report.when} of {report.head_line} " + ) + + notes.details += report.longreprtext + "\n" + + else: + raise RuntimeError("pytest_runtest_makereport received unknown test status") + + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """ + Reports the TAP result for this test item using our gathered TestNotes. + """ + res = yield + + assert notes_key in item.stash, "pgtap didn't annotate a test item?" + notes = item.stash[notes_key] + + if notes.failed: + tap.fail(item.nodeid, notes.details) + elif notes.skipped: + tap.skip(item.nodeid, notes.skip_reason) + else: + tap.ok(item.nodeid) + + return res -- 2.52.0