From 11759351232507bd290ae825d2a57f4af9fa88e0 Mon Sep 17 00:00:00 2001 From: Jacob Champion Date: Wed, 13 Aug 2025 10:58:56 -0700 Subject: [PATCH v1 2/6] Add support for pytest test suites Specify --enable-pytest/-Dpytest=enabled at configure time. This contains no Postgres test logic -- it is just a "vanilla" pytest skeleton. I've written a custom pgtap output plugin, used by the Meson mtest runner, to fully control what we see during CI test failures. The pytest-tap plugin would have been preferable, but it's now in maintenance mode, and it has problems with accidentally suppressing important collection failures. test_something.py is intended to show a sample failure in the CI. TODOs: - check_pytest.py has a known issue: it checks the Python interpreter linked to PL/Python rather than the Python interpreter in use by pytest. Perhaps the check script should just go away? - OpenBSD has an ANSI-related terminal bug, but I'm not sure if the bug is in Cirrus, the image, pytest, Python, or readline. The TERM envvar is unset to work around it. If this workaround is removed, a bad ANSI escape is inserted into the pgtap output and mtest is unable to parse it. - The Chocolatey CI setup is subpar. Need to find a way to bless the dependencies in use rather than pulling from pip... or maybe that will be done by the image baker. --- .cirrus.tasks.yml | 38 +++-- .gitignore | 1 + config/check_pytest.py | 138 ++++++++++++++++++ config/pytest-requirements.txt | 22 +++ configure | 170 ++++++++++++++++++++++- configure.ac | 30 +++- meson.build | 88 ++++++++++++ meson_options.txt | 8 +- pytest.ini | 1 + src/Makefile.global.in | 23 +++ src/makefiles/meson.build | 2 + src/test/Makefile | 11 +- src/test/meson.build | 1 + src/test/pytest/Makefile | 20 +++ src/test/pytest/README | 1 + src/test/pytest/meson.build | 16 +++ src/test/pytest/plugins/pgtap.py | 193 ++++++++++++++++++++++++++ src/test/pytest/pyt/test_something.py | 17 +++ 18 files changed, 765 insertions(+), 15 deletions(-) create mode 100644 config/check_pytest.py create mode 100644 config/pytest-requirements.txt create mode 100644 pytest.ini create mode 100644 src/test/pytest/Makefile create mode 100644 src/test/pytest/README create mode 100644 src/test/pytest/meson.build create mode 100644 src/test/pytest/plugins/pgtap.py create mode 100644 src/test/pytest/pyt/test_something.py diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index eca9d62fc22..80f9b394bd2 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -21,7 +21,8 @@ env: # target to test, for all but windows CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS - CHECKFLAGS: -Otarget + # TODO were we avoiding --keep-going on purpose? + CHECKFLAGS: -Otarget --keep-going PROVE_FLAGS: --timer # Build test dependencies as part of the build step, to see compiler # errors/warnings in one place. @@ -44,6 +45,7 @@ env: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled -Dplperl=enabled -Dplpython=enabled -Ddocs=enabled @@ -222,7 +224,9 @@ task: chown root:postgres /tmp/cores sysctl kern.corefile='/tmp/cores/%N.%P.core' setup_additional_packages_script: | - #pkg install -y ... + pkg install -y \ + py311-packaging \ + py311-pytest # NB: Intentionally build without -Dllvm. The freebsd image size is already # large enough to make VM startup slow, and even without llvm freebsd @@ -311,7 +315,10 @@ task: -Dpam=enabled setup_additional_packages_script: | - #pkgin -y install ... + pkgin -y install \ + py312-packaging \ + py312-test + ln -s /usr/pkg/bin/pytest-3.12 /usr/pkg/bin/pytest <<: *netbsd_task_template - name: OpenBSD - Meson @@ -322,6 +329,7 @@ task: OS_NAME: openbsd IMAGE_FAMILY: pg-ci-openbsd-postgres PKGCONFIG_PATH: '/usr/lib/pkgconfig:/usr/local/lib/pkgconfig' + TERM: # TODO why does pytest print ANSI escapes on OpenBSD? MESON_FEATURES: >- -Dbsd_auth=enabled @@ -330,7 +338,9 @@ task: -Duuid=e2fs setup_additional_packages_script: | - #pkg_add -I ... + pkg_add -I \ + py3-test \ + py3-packaging # Always core dump to ${CORE_DUMP_DIR} set_core_dump_script: sysctl -w kern.nosuidcoredump=2 <<: *openbsd_task_template @@ -489,8 +499,10 @@ task: EOF setup_additional_packages_script: | - #apt-get update - #DEBIAN_FRONTEND=noninteractive apt-get -y install ... + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + python3-pytest \ + python3-packaging matrix: # SPECIAL: @@ -513,14 +525,15 @@ task: su postgres <<-EOF ./configure \ --enable-cassert --enable-injection-points --enable-debug \ - --enable-tap-tests --enable-nls \ + --enable-tap-tests --enable-pytest --enable-nls \ --with-segsize-blocks=6 \ --with-libnuma \ --with-liburing \ \ ${LINUX_CONFIGURE_FEATURES} \ \ - CLANG="ccache clang-16" + CLANG="ccache clang-16" \ + PYTEST="env LD_PRELOAD=/lib/x86_64-linux-gnu/libasan.so.8 pytest" EOF build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin" upload_caches: ccache @@ -650,6 +663,8 @@ task: p5.34-io-tty p5.34-ipc-run python312 + py312-packaging + py312-pytest tcl zstd @@ -699,6 +714,7 @@ task: sh src/tools/ci/ci_macports_packages.sh $MACOS_PACKAGE_LIST # system python doesn't provide headers sudo /opt/local/bin/port select python3 python312 + sudo /opt/local/bin/port select pytest pytest312 # Make macports install visible for subsequent steps echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV upload_caches: macports @@ -772,6 +788,8 @@ task: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled + -DPYTEST=c:\Windows\system32\config\systemprofile\AppData\Roaming\Python\Python310\Scripts\pytest.exe -Dplperl=enabled -Dplpython=enabled @@ -780,8 +798,10 @@ task: depends_on: SanityCheck only_if: $CI_WINDOWS_ENABLED + # XXX Does Chocolatey really not have any Python package installers? setup_additional_packages_script: | REM choco install -y --no-progress ... + pip3 install --user packaging pytest setup_hosts_file_script: | echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts @@ -844,7 +864,7 @@ task: folder: ${CCACHE_DIR} setup_additional_packages_script: | - REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ... + C:\msys64\usr\bin\pacman.exe -S --noconfirm mingw-w64-ucrt-x86_64-python-packaging mingw-w64-ucrt-x86_64-python-pytest mingw_info_script: | %BASH% -c "where gcc" diff --git a/.gitignore b/.gitignore index 4e911395fe3..268426003b1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ win32ver.rc *.exe lib*dll.def lib*.pc +__pycache__/ # Local excludes in root directory /GNUmakefile diff --git a/config/check_pytest.py b/config/check_pytest.py new file mode 100644 index 00000000000..14b2f3eec9b --- /dev/null +++ b/config/check_pytest.py @@ -0,0 +1,138 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group +# +# Verify that pytest-requirements.txt is satisfied. This would probably be +# easier with pip, but requiring pip on build machines is a non-starter for +# many. +# +# The design philosophy of this script is to bend over backwards to help people +# figure out what is missing. The target audience for error output is the +# buildfarm operator who just wants to get the tests running, not the test +# developer who presumably already knows how to solve these problems. + +import sys +from typing import List # TODO: Python 3.9 will remove the need for this + + +def main(): + if len(sys.argv) != 2: + sys.exit("usage: python {} REQUIREMENTS_FILE".format(sys.argv[0])) + + requirements_file = sys.argv[1] + with open(requirements_file, "r") as f: + requirements = f.readlines() + + found = packaging_check(requirements) + if not found: + sys.exit("See src/test/pytest/README for package installation help.") + + +def packaging_check(requirements: List[str]) -> bool: + """ + The preferred dependency check, which unfortunately needs newer Python + facilities. Returns True if all dependencies were found. + """ + try: + # First, attempt to find importlib.metadata. This is part of the + # standard library from 3.8 onwards. Earlier Python versions have an + # official backport called importlib_metadata, which can generally be + # installed as a separate OS package (e.g. python3-importlib-metadata). + # This complication can be removed once we stop supporting Python 3.7. + try: + from importlib import metadata + except ImportError: + import importlib_metadata as metadata + + # packaging contains the PyPA definitions of requirement specifiers. + # This is again contained in a separate OS package (for example, + # python3-packaging). + import packaging + from packaging.requirements import Requirement + + except ImportError as err: + # We don't even have enough prerequisites to check our prerequisites. + # Try to fall back on the deprecated parser, to get a better error + # message. + found = setuptools_fallback(requirements) + + if not found: + # Well, the best we can do is just print the import error as-is. + print(err, file=sys.stderr) + + return False + + # Strip extraneous whitespace, whole-line comments, and empty lines from our + # specifier list. + requirements = [r.strip() for r in requirements] + requirements = [r for r in requirements if r and r[0] != "#"] + + found = True + for spec in requirements: + req = Requirement(spec) + + # Skip any packages marked as unneeded for this particular Python env. + if req.marker and not req.marker.evaluate(): + continue + + # Make sure the package is installed... + try: + version = metadata.version(req.name) + except metadata.PackageNotFoundError: + print("Package '{}' is not installed".format(req.name), file=sys.stderr) + found = False + continue + + # ...and that it has a compatible version. + if not req.specifier.contains(version): + print( + "Package '{}' has version {}, but '{}' is required".format( + req.name, version, req.specifier + ), + file=sys.stderr, + ) + found = False + continue + + return found + + +def setuptools_fallback(requirements: List[str]) -> bool: + """ + An alternative dependency helper, based on the old deprecated pkg_resources + module in setuptools, which is pretty widely available in older Pythons. The + point of this is to bootstrap the user into an environment that can run the + packaging_check(). + + Returns False if pkg_resources is also unavailable, in which case we just + have to do our best. + """ + try: + import pkg_resources + except ModuleNotFoundError: + return False + + # An extra newline makes the Autoconf output easier to read. + print(file=sys.stderr) + + # Go one-by-one through the requirements, printing each missing dependency. + found = True + for r in requirements: + try: + pkg_resources.require(r) + except pkg_resources.DistributionNotFound as err: + # The error descriptions given here are pretty good as-is. + print(err, file=sys.stderr) + found = False + except pkg_resources.RequirementParseError as err: + assert False # TODO + + # The only reason the fallback would be called is if we're missing required + # packages. So if we "found them", the requirements file is broken... + assert ( + not found + ), "setuptools_fallback() succeeded unexpectedly; is the requirements file incomplete?" + + return True + + +if __name__ == "__main__": + main() diff --git a/config/pytest-requirements.txt b/config/pytest-requirements.txt new file mode 100644 index 00000000000..157262a684f --- /dev/null +++ b/config/pytest-requirements.txt @@ -0,0 +1,22 @@ +# +# This file contains the Python packages which are required in order for us to +# enable pytest. +# +# The syntax is a *subset* of pip's requirements.txt syntax, so that both pip +# and check_pytest.py can use it. Only whole-line comments and standard Python +# dependency specifiers are allowed. pip-specific goodies like includes and +# environment substitutions are not supported; keep it simple. +# +# Packages belong here if their absence should cause a configuration failure. If +# you'd like to make a package optional, consider using pytest.importorskip() +# instead. +# + +# pytest 7.0 was the last version which supported Python 3.6, but the BSDs have +# started putting 8.x into ports, so we support both. (pytest 8 can be used +# throughout once we drop support for Python 3.7.) +pytest >= 7.0, < 9 + +# These are meta-packages which allow check_pytest.py to run. +packaging +importlib_metadata ; python_version < "3.8" diff --git a/configure b/configure index 39c68161cec..860b07763dc 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ vpath_build PG_SYSROOT PG_VERSION_NUM LDFLAGS_EX_BE +PYTEST PROVE DBTOEPUB FOP @@ -773,6 +774,7 @@ CFLAGS CC enable_injection_points PG_TEST_EXTRA +enable_pytest enable_tap_tests enable_dtrace DTRACEFLAGS @@ -851,6 +853,7 @@ enable_profiling enable_coverage enable_dtrace enable_tap_tests +enable_pytest enable_injection_points with_blocksize with_segsize @@ -1551,7 +1554,10 @@ Optional Features: --enable-profiling build with profiling enabled --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support - --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) + --enable-tap-tests enable (Perl-based) TAP tests (requires Perl and + IPC::Run) + --enable-pytest enable (Python-based) pytest suites (requires + Python) --enable-injection-points enable injection points (for testing) --enable-depend turn on automatic dependency tracking @@ -3639,7 +3645,7 @@ fi # -# TAP tests +# Test frameworks # @@ -3667,6 +3673,32 @@ fi + +# Check whether --enable-pytest was given. +if test "${enable_pytest+set}" = set; then : + enableval=$enable_pytest; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-pytest option" "$LINENO" 5 + ;; + esac + +else + enable_pytest=no + +fi + + + + + + # # Injection points # @@ -19120,6 +19152,140 @@ $as_echo "$modulestderr" >&6; } fi fi +if test "$enable_pytest" = yes; then + # Mirror the prove checks, above, for pytest. We don't require the user to + # have selected --with-python, but we do need a Python installation. + if test -z "$PYTHON"; then + if test -z "$PYTHON"; then + for ac_prog in python3 python +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTHON+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTHON in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTHON="$PYTHON" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTHON="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTHON=$ac_cv_path_PYTHON +if test -n "$PYTHON"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON" >&5 +$as_echo "$PYTHON" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTHON" && break +done + +else + # Report the value of PYTHON in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTHON" >&5 +$as_echo_n "checking for PYTHON... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON" >&5 +$as_echo "$PYTHON" >&6; } +fi + +if test x"$PYTHON" = x""; then + as_fn_error $? "Python not found" "$LINENO" 5 +fi + + fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python packages required for pytest" >&5 +$as_echo_n "checking for Python packages required for pytest... " >&6; } + modulestderr=`"$PYTHON" "$srcdir/config/check_pytest.py" "$srcdir/config/pytest-requirements.txt" 2>&1 >/dev/null` + if test $? -eq 0; then + echo "$modulestderr" >&5 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $modulestderr" >&5 +$as_echo "$modulestderr" >&6; } + as_fn_error $? "Additional Python packages are required to run the pytest suites" "$LINENO" 5 + fi + if test -z "$PYTEST"; then + for ac_prog in pytest py.test +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTEST+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTEST in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTEST="$PYTEST" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTEST="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTEST=$ac_cv_path_PYTEST +if test -n "$PYTEST"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTEST" && break +done + +else + # Report the value of PYTEST in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTEST" >&5 +$as_echo_n "checking for PYTEST... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +fi + + if test -z "$PYTEST"; then + as_fn_error $? "pytest not found" "$LINENO" 5 + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/configure.ac b/configure.ac index 066e3976c0a..f4bf94a078f 100644 --- a/configure.ac +++ b/configure.ac @@ -231,11 +231,16 @@ AC_SUBST(DTRACEFLAGS)]) AC_SUBST(enable_dtrace) # -# TAP tests +# Test frameworks # PGAC_ARG_BOOL(enable, tap-tests, no, - [enable TAP tests (requires Perl and IPC::Run)]) + [enable (Perl-based) TAP tests (requires Perl and IPC::Run)]) AC_SUBST(enable_tap_tests) + +PGAC_ARG_BOOL(enable, pytest, no, + [enable (Python-based) pytest suites (requires Python)]) +AC_SUBST(enable_pytest) + AC_ARG_VAR(PG_TEST_EXTRA, [enable selected extra tests (overridden at runtime by PG_TEST_EXTRA environment variable)]) @@ -2442,6 +2447,27 @@ if test "$enable_tap_tests" = yes; then fi fi +if test "$enable_pytest" = yes; then + # Mirror the prove checks, above, for pytest. We don't require the user to + # have selected --with-python, but we do need a Python installation. + if test -z "$PYTHON"; then + PGAC_PATH_PYTHON + fi + AC_MSG_CHECKING(for Python packages required for pytest) + [modulestderr=`"$PYTHON" "$srcdir/config/check_pytest.py" "$srcdir/config/pytest-requirements.txt" 2>&1 >/dev/null`] + if test $? -eq 0; then + echo "$modulestderr" >&AS_MESSAGE_LOG_FD + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT([$modulestderr]) + AC_MSG_ERROR([Additional Python packages are required to run the pytest suites]) + fi + PGAC_PATH_PROGS(PYTEST, pytest py.test) + if test -z "$PYTEST"; then + AC_MSG_ERROR([pytest not found]) + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/meson.build b/meson.build index 2598758f6d3..5166d82e607 100644 --- a/meson.build +++ b/meson.build @@ -1699,6 +1699,35 @@ endif +############################################################### +# Library: pytest +############################################################### + +pytest_enabled = false +pytest = not_found_dep + +pytestopt = get_option('pytest') +if not pytestopt.disabled() + pytest_check = run_command(python, 'config/check_pytest.py', + 'config/pytest-requirements.txt', check: false) + if pytest_check.returncode() != 0 + message(pytest_check.stderr().strip()) + if pytestopt.enabled() + error('Additional Python packages are required to run the pytest suites.') + else + warning('Additional Python packages are required to run the pytest suites.') + endif + endif + + pytest = find_program(get_option('PYTEST'), native: true, required: pytestopt) + + if pytest.found() and pytest_check.returncode() == 0 + pytest_enabled = true + endif +endif + + + ############################################################### # Library: zstd ############################################################### @@ -3776,6 +3805,63 @@ foreach test_dir : tests ) endforeach install_suites += test_group + elif kind == 'pytest' + testwrap_pytest = testwrap_base + if not pytest_enabled + testwrap_pytest += ['--skip', 'pytest not enabled'] + endif + + test_command = [ + pytest.full_path(), + '-c', meson.project_source_root() / 'pytest.ini', + '--verbose', + '-p', 'pgtap', # enable our test reporter plugin + '-ra', # show skipped and xfailed tests too + ] + + # Add temporary install, the build directory for non-installed binaries and + # also test/ for non-installed test binaries built separately. + env = test_env + env.prepend('PATH', temp_install_bindir, test_dir['bd'], test_dir['bd'] / 'test') + temp_install_datadir = '@0@@1@'.format(test_install_destdir, dir_prefix / dir_data) + env.set('share_contrib_dir', temp_install_datadir / 'contrib') + env.prepend('PYTHONPATH', meson.project_source_root() / 'src' / 'test' / 'pytest' / 'plugins') + + foreach name, value : t.get('env', {}) + env.set(name, value) + endforeach + + test_group = test_dir['name'] + test_kwargs = { + 'protocol': 'tap', + 'suite': test_group, + 'timeout': 1000, + 'depends': test_deps + t.get('deps', []), + 'env': env, + } + t.get('test_kwargs', {}) + + foreach onetest : t['tests'] + # Make test names prettier, remove pyt/ and .py + onetest_p = onetest + if onetest_p.startswith('pyt/') + onetest_p = onetest.split('pyt/')[1] + endif + if onetest_p.endswith('.py') + onetest_p = fs.stem(onetest_p) + endif + + test(test_dir['name'] / onetest_p, + python, + kwargs: test_kwargs, + args: testwrap_pytest + [ + '--testgroup', test_dir['name'], + '--testname', onetest_p, + '--', test_command, + test_dir['sd'] / onetest, + ], + ) + endforeach + install_suites += test_group else error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd'])) endif @@ -3950,6 +4036,7 @@ summary( 'dtrace': dtrace, 'flex': '@0@ @1@'.format(flex.full_path(), flex_version), 'prove': prove, + 'pytest': pytest, }, section: 'Programs', ) @@ -3990,6 +4077,7 @@ summary( summary( { 'tap': tap_tests_enabled, + 'pytest': pytest_enabled, }, section: 'Other features', list_sep: ' ', diff --git a/meson_options.txt b/meson_options.txt index 06bf5627d3c..88f22e699d9 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -41,7 +41,10 @@ option('cassert', type: 'boolean', value: false, description: 'Enable assertion checks (for debugging)') option('tap_tests', type: 'feature', value: 'auto', - description: 'Enable TAP tests') + description: 'Enable (Perl-based) TAP tests') + +option('pytest', type: 'feature', value: 'auto', + description: 'Enable (Python-based) pytest suites') option('injection_points', type: 'boolean', value: false, description: 'Enable injection points') @@ -195,6 +198,9 @@ option('PERL', type: 'string', value: 'perl', option('PROVE', type: 'string', value: 'prove', description: 'Path to prove binary') +option('PYTEST', type: 'array', value: ['pytest', 'py.test'], + description: 'Path to pytest binary') + option('PYTHON', type: 'array', value: ['python3', 'python'], description: 'Path to python binary') diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..eea2c180278 --- /dev/null +++ b/pytest.ini @@ -0,0 +1 @@ +[pytest] diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 8b1b357beaa..fc744166bd2 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -211,6 +211,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_pytest = @enable_pytest@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ @@ -354,6 +355,7 @@ MSGFMT = @MSGFMT@ MSGFMT_FLAGS = @MSGFMT_FLAGS@ MSGMERGE = @MSGMERGE@ OPENSSL = @OPENSSL@ +PYTEST = @PYTEST@ PYTHON = @PYTHON@ TAR = @TAR@ XGETTEXT = @XGETTEXT@ @@ -508,6 +510,27 @@ prove_installcheck = @echo "TAP tests not enabled. Try configuring with --enable prove_check = $(prove_installcheck) endif +ifeq ($(enable_pytest),yes) + +pytest_installcheck = @echo "Installcheck is not currently supported for pytest." + +define pytest_check +echo "# +++ pytest check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest/plugins:$$PYTHONPATH' \ + $(with_temp_install) \ + $(PYTEST) -c '$(abs_top_srcdir)/pytest.ini' --verbose -ra ./pyt/ +endef + +else +pytest_installcheck = @echo "pytest is not enabled. Try configuring with --enable-pytest" +pytest_check = $(pytest_installcheck) +endif + # Installation. install_bin = @install_bin@ diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 54dbc059ada..f69eb1068db 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -56,6 +56,7 @@ pgxs_kv = { 'enable_nls': libintl.found() ? 'yes' : 'no', 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', 'enable_tap_tests': tap_tests_enabled ? 'yes' : 'no', + 'enable_pytest': pytest_enabled ? 'yes' : 'no', 'enable_debug': get_option('debug') ? 'yes' : 'no', 'enable_coverage': 'no', 'enable_dtrace': dtrace.found() ? 'yes' : 'no', @@ -147,6 +148,7 @@ pgxs_bins = { 'OPENSSL': openssl, 'PERL': perl, 'PROVE': prove, + 'PYTEST': pytest, 'PYTHON': python, 'TAR': tar, 'ZSTD': program_zstd, diff --git a/src/test/Makefile b/src/test/Makefile index 511a72e6238..0be9771d71f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -12,7 +12,16 @@ subdir = src/test top_builddir = ../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription +SUBDIRS = \ + authentication \ + isolation \ + modules \ + perl \ + postmaster \ + pytest \ + recovery \ + regress \ + subscription ifeq ($(with_icu),yes) SUBDIRS += icu diff --git a/src/test/meson.build b/src/test/meson.build index ccc31d6a86a..d08a6ef61c2 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -5,6 +5,7 @@ subdir('isolation') subdir('authentication') subdir('postmaster') +subdir('pytest') subdir('recovery') subdir('subscription') subdir('modules') diff --git a/src/test/pytest/Makefile b/src/test/pytest/Makefile new file mode 100644 index 00000000000..2bdca96ccbe --- /dev/null +++ b/src/test/pytest/Makefile @@ -0,0 +1,20 @@ +#------------------------------------------------------------------------- +# +# Makefile for pytest +# +# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/pytest/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/pytest +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(pytest_check) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/pytest/README b/src/test/pytest/README new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/src/test/pytest/README @@ -0,0 +1 @@ +TODO diff --git a/src/test/pytest/meson.build b/src/test/pytest/meson.build new file mode 100644 index 00000000000..abd128dfa24 --- /dev/null +++ b/src/test/pytest/meson.build @@ -0,0 +1,16 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +if not pytest_enabled + subdir_done() +endif + +tests += { + 'name': 'pytest', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'pytest': { + 'tests': [ + 'pyt/test_something.py', + ], + }, +} diff --git a/src/test/pytest/plugins/pgtap.py b/src/test/pytest/plugins/pgtap.py new file mode 100644 index 00000000000..ef8291e291c --- /dev/null +++ b/src/test/pytest/plugins/pgtap.py @@ -0,0 +1,193 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +import os +import sys +from typing import Optional + +import pytest + +# +# Helpers +# + + +class TAP: + """ + A basic API for reporting via the TAP protocol. + """ + + def __init__(self): + self.count = 0 + + # XXX interacts poorly with testwrap's boilerplate diagnostics + # self.print("TAP version 13") + + def expect(self, num: int): + self.print(f"1..{num}") + + def print(self, *args): + print(*args, file=sys.__stdout__) + + def ok(self, name: str): + self.count += 1 + self.print("ok", self.count, "-", name) + + def skip(self, name: str, reason: str): + self.count += 1 + self.print("ok", self.count, "-", name, "# skip", reason) + + def fail(self, name: str, details: str): + self.count += 1 + self.print("not ok", self.count, "-", name) + + # mtest has some odd behavior around TAP tests where it won't print + # diagnostics on failure if they're part of the stdout stream, so we + # might as well just dump the details directly to stderr instead. + print(details, file=sys.__stderr__) + + +tap = TAP() + + +class TestNotes: + """ + Annotations for a single test. The existing pytest hooks keep interesting + information somewhat separated across the different stages + (setup/test/teardown), so this class is used to correlate them. + """ + + skipped = False + skip_reason = None + + failed = False + details = "" + + +# Register a custom key in the stash dictionary for keeping our TestNotes. +notes_key = pytest.StashKey[TestNotes]() + + +# +# Hook Implementations +# + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + """ + Hijacks the standard streams as soon as possible during pytest startup. The + pytest-formatted output gets logged to file instead, and we'll use the + original sys.__stdout__/__stderr__ streams for the TAP protocol. + """ + logdir = os.getenv("TESTLOGDIR") + if not logdir: + raise RuntimeError("pgtap requires the TESTLOGDIR envvar to be set") + + os.makedirs(logdir) + logpath = os.path.join(logdir, "pytest.log") + sys.stdout = sys.stderr = open(logpath, "a", buffering=1) + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session, exitstatus): + """ + Suppresses nonzero exit codes due to failed tests. (In that case, we want + Meson to report a failure count, not a generic ERROR.) + """ + if exitstatus == pytest.ExitCode.TESTS_FAILED: + session.exitstatus = pytest.ExitCode.OK + + +@pytest.hookimpl +def pytest_collectreport(report): + # Include collection failures directly in Meson error output. + if report.failed: + print(report.longreprtext, file=sys.__stderr__) + + +@pytest.hookimpl +def pytest_internalerror(excrepr, excinfo): + # Include internal errors directly in Meson error output. + print(excrepr, file=sys.__stderr__) + + +# +# Hook Wrappers +# +# In pytest parlance, a "wrapper" for a hook can inspect and optionally modify +# existing hooks' behavior, but it does not replace the hook chain. This is done +# through a generator-style API which chains the hooks together (see the use of +# `yield`). +# + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collection(session): + """Reports the number of gathered tests after collection is finished.""" + res = yield + tap.expect(session.testscollected) + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Annotates a test item with our TestNotes and grabs relevant information for + reporting. + + This is called multiple times per test, so it's not correct to print the TAP + result here. (A test and its teardown stage can both fail, and we want to + see the details for both.) We instead combine all the information for use by + our pytest_runtest_protocol wrapper later on. + """ + res = yield + + if notes_key not in item.stash: + item.stash[notes_key] = TestNotes() + notes = item.stash[notes_key] + + report = res.get_result() + if report.passed: + pass # no annotation needed + + elif report.skipped: + notes.skipped = True + _, _, notes.skip_reason = report.longrepr + + elif report.failed: + notes.failed = True + + if not notes.details: + notes.details += "{:_^72}\n\n".format(f" {report.head_line} ") + + if report.when in ("setup", "teardown"): + notes.details += "\n{:_^72}\n\n".format( + f" Error during {report.when} of {report.head_line} " + ) + + notes.details += report.longreprtext + "\n" + + else: + raise RuntimeError("pytest_runtest_makereport received unknown test status") + + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """ + Reports the TAP result for this test item using our gathered TestNotes. + """ + res = yield + + assert notes_key in item.stash, "pgtap didn't annotate a test item?" + notes = item.stash[notes_key] + + if notes.failed: + tap.fail(item.nodeid, notes.details) + elif notes.skipped: + tap.skip(item.nodeid, notes.skip_reason) + else: + tap.ok(item.nodeid) + + return res diff --git a/src/test/pytest/pyt/test_something.py b/src/test/pytest/pyt/test_something.py new file mode 100644 index 00000000000..5bd45618512 --- /dev/null +++ b/src/test/pytest/pyt/test_something.py @@ -0,0 +1,17 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +import pytest + + +@pytest.fixture +def hey(): + yield + raise "uh-oh" + + +def test_something(hey): + assert 2 == 4 + + +def test_something_else(): + assert 2 == 2 -- 2.34.1