postgresql-jdbc.git / summary / log / commit / refs
commit 0bb6455e074737dedd0200c55f21cf9cc03e2fb1
Author: Sehrope Sarkuni <sehrope@jackdb.com>
Date: Tue May 18 10:29:26 2021 +0000
test: Improve omni action job names and split replication to run standalone
Improves names of omni actions jobs and present them in a sorted manner so that all the
experimental jobs appear last.
Also changes the job execution so that a single job performs the replication test on their
own. Running them combined with other tests causes them to randomly fail and this is a
hopefully temporary measure to get the tests running successfully even if it is in isolation.
Gradle args as a dispatch parameter have been removed. All jobs now run test suite with code
coverage and enable Gradle build scans.
By default slow and replication tests are not run on the majority of matrix jobs. This can be
overridden with the new "Default test group" input to specify which tests they will be running:
fast - Only fast tests
slow - Fast and slow tests
all - All tests including replication tests
.github/workflows/omni.yml | 418 +++++++++++++++++++++++----------------------
1 file changed, 216 insertions(+), 202 deletions(-)
diff --git a/.github/workflows/omni.yml b/.github/workflows/omni.yml
index 7140c1d3..fbeb4ea1 100644
--- a/.github/workflows/omni.yml
+++ b/.github/workflows/omni.yml
@@ -3,10 +3,6 @@ name: Omni CI
on:
workflow_dispatch:
inputs:
- gradle_args:
- description: Gradle arguments
- required: false
- default: "jandex test jacocoReport"
# Filters to limit generated matrix. Empty or "*" values are ignored.
matrix_jdk_distribution:
description: JDK Distribution (adopt, zulu, ...)
@@ -32,10 +28,10 @@ on:
description: SCRAM (true | false)
required: false
default: "*"
- matrix_slow_tests:
- description: Slow Tests (default | always | never)
+ matrix_default_test_group:
+ description: Default test group (fast | slow | all)
required: false
- default: default
+ default: fast
schedule:
- cron: "0 6 * * *"
@@ -52,203 +48,220 @@ jobs:
MATRIX_QUERY_MODE: '${{ github.event.inputs.matrix_query_mode }}'
MATRIX_SCRAM: '${{ github.event.inputs.matrix_scram }}'
MATRIX_SSL: '${{ github.event.inputs.matrix_ssl }}'
- MATRIX_SLOW_TESTS: '${{ github.event.inputs.matrix_slow_tests }}'
- steps:
- - id: set-matrix
- run: |
- node -e "
- const DEFAULT_SLOW_TESTS = (() => {
- if (process.env.MATRIX_SLOW_TESTS === 'always') {
+ MATRIX_DEFAULT_TEST_GROUP: '${{ github.event.inputs.matrix_default_test_group }}'
+ # Script as an environment variable so we don't have to worry shell substitions
+ MATRIX_GENERATION_NODE_JS_SCRIPT: |
+ const DEFAULT_TEST_GROUP = process.env.MATRIX_DEFAULT_TEST_GROUP || 'fast';
+
+ const JDK_OPTIONS = [
+ { version: '8', lts: true, distribution: 'zulu' },
+ { version: '11', lts: true, distribution: 'zulu' },
+ { version: '15', lts: false, distribution: 'zulu' },
+ ];
+
+ const OTHER_JDK_OPTIONS = [
+ // Adopt
+ { group: 'Adopt Hotspot', version: '8', lts: true, distribution: 'adopt-hotspot' },
+ { group: 'Adopt Hotspot', version: '11', lts: true, distribution: 'adopt-hotspot' },
+
+ // Adopt OpenJ9
+ // TODO: Replace these hard coded versions with something that dynamically picks the most recent
+ { group: 'Adopt OpenJ9', version: '8', lts: true, distribution: 'adopt-openj9'},
+ { group: 'Adopt OpenJ9', version: '11', lts: true, distribution: 'adopt-openj9'},
+
+ // Amazon Corretto
+ { group: 'Corretto', version: '8', lts: true, distribution: 'jdkfile', url: 'https://corretto.aws/downloads/latest/amazon-corretto-8-x64-linux-jdk.tar.gz'},
+ { group: 'Corretto', version: '11', lts: true, distribution: 'jdkfile', url: 'https://corretto.aws/downloads/latest/amazon-corretto-11-x64-linux-jdk.tar.gz'},
+ ];
+
+ const PG_VERSIONS = [
+ '8.4',
+ '9.0',
+ '9.1',
+ '9.2',
+ '9.3',
+ '9.4',
+ '9.5',
+ '9.6',
+ '10',
+ '11',
+ '12',
+ '13',
+ // TODO: Determine the latest and greatest server version automatically so we don't need to periodically update this
+ ];
+
+
+ const LTS_JDK_OPTIONS = JDK_OPTIONS.filter(jdk => jdk.lts);
+ const UNSTABLE_JDK_OPTIONS = JDK_OPTIONS.filter(jdk => !jdk.lts);
+ const LATEST_JDK = LTS_JDK_OPTIONS.slice(-1)[0];
+ const LATEST_PG_VERSION = PG_VERSIONS.slice(-1)[0];
+
+ const list = [];
+
+ const matchesMatrixFilter = (name, value) => {
+ const env_value = process.env['MATRIX_' + name];
+ if (!env_value || env_value === '*') {
return true;
- } else if (process.env.MATRIX_SLOW_TESTS === 'never') {
- return false;
}
- return undefined;
- })();
-
- const JDK_OPTIONS = [
- { version: '8', lts: true, distribution: 'zulu' },
- { version: '11', lts: true, distribution: 'zulu' },
- { version: '15', lts: false, distribution: 'zulu' },
- ];
-
- const OTHER_JDK_OPTIONS = [
- // Adopt
- { version: '8', lts: true, distribution: 'adopt' },
- { version: '11', lts: true, distribution: 'adopt' },
-
- // Adopt OpenJ9
- // TODO: Replace these hard coded versions with something that dynamically picks the most recent
- { version: '8', lts: true, distribution: 'jdkfile', url: 'https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u292-b10_openj9-0.26.0/OpenJDK8U-jdk_x64_linux_openj9_8u292b10_openj9-0.26.0.tar.gz'},
- { version: '11', lts: true, distribution: 'jdkfile', url: 'https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.11%2B9_openj9-0.26.0/OpenJDK11U-jdk_x64_linux_openj9_11.0.11_9_openj9-0.26.0.tar.gz'},
-
- // Amazon Corretto
- { version: '8', lts: true, distribution: 'jdkfile', url: 'https://corretto.aws/downloads/latest/amazon-corretto-8-x64-linux-jdk.tar.gz'},
- { version: '11', lts: true, distribution: 'jdkfile', url: 'https://corretto.aws/downloads/latest/amazon-corretto-11-x64-linux-jdk.tar.gz'},
- ];
-
- const PG_VERSIONS = [
- '8.4',
- '9.0',
- '9.1',
- '9.2',
- '9.3',
- '9.4',
- '9.5',
- '9.6',
- '10',
- '11',
- '12',
- '13',
- // TODO: Determine the latest and greatest server version automatically so we don't need to periodically update this
- ];
-
-
- const LTS_JDK_OPTIONS = JDK_OPTIONS.filter(jdk => jdk.lts);
- const UNSTABLE_JDK_OPTIONS = JDK_OPTIONS.filter(jdk => !jdk.lts);
- const LATEST_JDK = LTS_JDK_OPTIONS.slice(-1)[0];
- const LATEST_PG_VERSION = PG_VERSIONS.slice(-1)[0];
-
- const list = [];
-
- const matchesMatrixFilter = (name, value) => {
- const env_value = process.env['MATRIX_' + name];
- if (!env_value || env_value === '*') {
- return true;
- }
- // TODO: Consider expanding this to do globbing
- return env_value === ('' + value);
- };
-
- const addItem = (opts) => {
- const os = 'ubuntu-latest';
- const pg_version = opts.pg_version ?? LATEST_PG_VERSION;
- const jdk = opts.jdk ?? LATEST_JDK;
- const slow_tests = DEFAULT_SLOW_TESTS ?? opts.slow_tests ?? false;
- const replication_tests = opts.replication_tests ?? false;
-
- const isAtLeast = (minVersion) => Number(pg_version) >= Number(minVersion);
- const scramSupported = isAtLeast('10');
- const sslSupported = isAtLeast('9.3');
-
- const query_mode = opts.query_mode;
- const ssl = opts.ssl ?? sslSupported;
- const scram = opts.scram ?? scramSupported;
-
- if (!matchesMatrixFilter('JDK_DISTRIBUTION', jdk.distribution) ||
- !matchesMatrixFilter('JDK_VERSION', jdk.version) ||
- !matchesMatrixFilter('PG_VERSION', pg_version) ||
- !matchesMatrixFilter('QUERY_MODE', query_mode ?? '') ||
- !matchesMatrixFilter('SCRAM', scram) ||
- !matchesMatrixFilter('SSL', ssl)) {
- return;
- }
+ // TODO: Consider expanding this to do globbing
+ return env_value === ('' + value);
+ };
+
+ const addItem = (opts) => {
+ const name = opts.name ?? '';
+ const os = 'ubuntu-latest';
+ const pg_version = opts.pg_version ?? LATEST_PG_VERSION;
+ const jdk = opts.jdk ?? LATEST_JDK;
+ const experimental = opts.experimental ?? false;
+ const test_group = opts.test_group ?? DEFAULT_TEST_GROUP;
+
+ const isAtLeast = (minVersion) => Number(pg_version) >= Number(minVersion);
+ const scramSupported = isAtLeast('10');
+ const sslSupported = isAtLeast('9.3');
+
+ const query_mode = opts.query_mode;
+ const ssl = opts.ssl ?? sslSupported;
+ const scram = opts.scram ?? scramSupported;
+
+ if (!matchesMatrixFilter('JDK_DISTRIBUTION', jdk.distribution) ||
+ !matchesMatrixFilter('JDK_VERSION', jdk.version) ||
+ !matchesMatrixFilter('PG_VERSION', pg_version) ||
+ !matchesMatrixFilter('QUERY_MODE', query_mode ?? '') ||
+ !matchesMatrixFilter('SCRAM', scram) ||
+ !matchesMatrixFilter('SSL', ssl)) {
+ return;
+ }
- const tags = [];
- if (!slow_tests) {
- tags.push('!org.postgresql.test.SlowTests');
- }
- if (!replication_tests) {
- tags.push('!org.postgresql.test.Replication');
+ const junit_include_tags = (() => {
+ switch (test_group) {
+ case 'replication':
+ // Only replication tests
+ return 'org.postgresql.test.Replication';
+ case 'fast':
+ // Fast group does not run slow or replication tests
+ return '!org.postgresql.test.SlowTests & !org.postgresql.test.Replication';
+ case 'slow':
+ // Everything but replication tests (which includes all the really slow ones)
+ return '!org.postgresql.test.Replication';
+ case 'all':
+ // Everything
+ return ''
}
- const junit_include_tags = tags.join(' & ');
-
- list.push({
- os,
- jdk_version: jdk.version,
- jdk_distribution: jdk.distribution,
- jdk_url: jdk.url,
- pg_version,
- ssl,
- scram,
- server_tz: opts.server_tz ?? 'Etc/UTC',
- slow_tests,
- replication_tests,
- junit_include_tags,
- experimental: opts.experimental ?? false,
- query_mode,
- gradle_args: opts.gradle_args ?? '',
- });
- };
-
- // Latest JDK x each stable PG version
- for (const pg_version of PG_VERSIONS) {
- addItem({
- pg_version,
- });
- }
-
- // Latest PG version x each remaining JDK
- for (const jdk of JDK_OPTIONS) {
- if (jdk == LATEST_JDK) {
- continue; // Skip duplicate
- }
- addItem({
- jdk,
- pg_version: LATEST_PG_VERSION,
- experimental: !jdk.lts,
- });
- }
-
- // No SSL / No SCRAM (only on latest PG / JDK)
- addItem({
- ssl: false,
- scram: false,
- });
-
- // Custom server timezones (only on latest PG / JDK)
- addItem({
- server_tz: 'America/New_York'
- });
- addItem({
- server_tz: 'Pacific/Chatham'
- });
-
- // Custom query modes (only on latest PG / JDK)
- addItem({
- query_mode: 'simple',
- });
- addItem({
- query_mode: 'extendedForPrepared',
- });
- addItem({
- query_mode: 'extendedCacheEverything',
- });
-
- // Slow and replication tests (only on latest PG / JDK)
- // NOTE: Marked experimental for now due to server issues
- addItem({
- slow_tests: true,
- replication_tests: true,
- experimental: true,
- gradle_args: '--scan',
- });
-
- // TODO: Add latest PG built from source marked as experimental
- for(const jdk of OTHER_JDK_OPTIONS) {
- addItem({
- jdk,
- pg_version: LATEST_PG_VERSION,
- experimental: !jdk.lts,
- });
- }
-
- if (list.length === 0) {
- throw new Error('Matrix list is empty. Check you matrix filters to ensure they match a valid combination.');
- }
-
- const toKey = (item) => JSON.stringify(Object.entries(item).sort())
- const include = list.filter((item, i) => {
- const key = toKey(item);
- // Deduplicate by picking only the first entry matching this key
- return i === list.findIndex((other) => key === toKey(other));
- });
-
- console.log('::set-output name=matrix::' + JSON.stringify({ include }));
- "
+ throw new Error('Invalid test group: ' + test_group);
+ })();
+
+ list.push({
+ name: [
+ experimental ? 'Experimental' : '',
+ name,
+ `${jdk.group} ${jdk.version} x PG ${pg_version}`
+ ].filter(x => x).join(' - '),
+ os,
+ jdk_version: jdk.version,
+ jdk_distribution: jdk.distribution,
+ jdk_url: jdk.url,
+ pg_version,
+ ssl,
+ scram,
+ server_tz: opts.server_tz ?? 'Etc/UTC',
+ experimental,
+ junit_include_tags,
+ query_mode,
+ });
+ };
+
+ // Latest JDK x each stable PG version
+ for (const pg_version of PG_VERSIONS) {
+ addItem({
+ pg_version,
+ });
+ }
+
+ // Latest PG version x each remaining JDK
+ for (const jdk of JDK_OPTIONS) {
+ if (jdk == LATEST_JDK) {
+ continue; // Skip duplicate
+ }
+ addItem({
+ jdk,
+ pg_version: LATEST_PG_VERSION,
+ experimental: !jdk.lts,
+ });
+ }
+
+ // No SSL / No SCRAM (only on latest PG / JDK)
+ addItem({
+ name: `No SSL / No SCRAM`,
+ ssl: false,
+ scram: false,
+ });
+
+ // Custom server timezones (only on latest PG / JDK)
+ addItem({
+ name: `Server TZ - America/New_York`,
+ server_tz: 'America/New_York'
+ });
+ addItem({
+ name: `Server TZ - Pacific/Chatham`,
+ server_tz: 'Pacific/Chatham'
+ });
+
+ // Custom query modes (only on latest PG / JDK)
+ addItem({
+ name: `Query Mode - simple`,
+ query_mode: 'simple',
+ });
+ addItem({
+ name: `Query Mode - extendedForPrepared`,
+ query_mode: 'extendedForPrepared',
+ });
+ addItem({
+ name: `Query Mode - extendedCacheEverything`,
+ query_mode: 'extendedCacheEverything',
+ });
+
+ // Slow tests (only on latest PG / JDK)
+ addItem({
+ name: `Slow Tests`,
+ test_group: 'slow',
+ });
+ // Replication tests (only on latest PG / JDK)
+ addItem({
+ name: `Replication Tests`,
+ test_group: 'replication',
+ });
+
+ // TODO: Add latest PG built from source marked as experimental
+
+ for(const jdk of OTHER_JDK_OPTIONS) {
+ addItem({
+ jdk,
+ pg_version: LATEST_PG_VERSION,
+ experimental: !jdk.lts,
+ });
+ }
+
+ if (list.length === 0) {
+ throw new Error('Matrix list is empty. Check you matrix filters to ensure they match a valid combination.');
+ }
+
+ const toKey = (item) => JSON.stringify(Object.entries(item).filter(([field]) => field != 'name').sort())
+ const include = list.filter((item, i) => {
+ const key = toKey(item);
+ // Deduplicate by picking only the first entry matching this key
+ return i === list.findIndex((other) => key === toKey(other));
+ });
+ // Sort so that all the experimental jobs appear at the end of the list
+ include.sort((a, b) => (a.experimental ? 1 : 0) - (b.experimental ? 1 : 0))
+
+ console.log('::set-output name=matrix::' + JSON.stringify({ include }));
+ steps:
+ - id: set-matrix
+ run: |
+ node -e "${MATRIX_GENERATION_NODE_JS_SCRIPT}"
test:
- name: '${{ toJson(matrix) }}'
+ name: '${{ matrix.name }}'
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.experimental }}
needs: matrix_prep
@@ -263,6 +276,10 @@ jobs:
SCRAM: ${{ matrix.scram }}
TZ: ${{ matrix.server_tz }}
steps:
+ - name: 'Show Matrix'
+ env:
+ MATRIX_JSON: ${{ toJSON(matrix) }}
+ run: echo "${MATRIX_JSON}"
- uses: actions/checkout@v2
with:
fetch-depth: 50
@@ -321,7 +338,6 @@ jobs:
cat <<EOF >build.local.properties
preferQueryMode=${{ matrix.query_mode}}
EOF
-
- name: Test
uses: burrunan/gradle-cache-action@v1
env:
@@ -329,12 +345,10 @@ jobs:
S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.S3_BUILD_CACHE_SECRET_KEY }}
with:
job-id: jdk${{ matrix.jdk_version }}_pg${{ matrix.pg_version}}
- arguments: ${{ matrix.gradle_args }} --no-parallel --no-daemon ${{ github.event.inputs.gradle_args }}
+ arguments: --no-parallel --no-daemon --scan jandex test jacocoReport
properties: |
includeTestTags=${{ matrix.junit_include_tags }}
-
- name: 'Upload code coverage'
uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
- if: ${{ contains(github.event.inputs.gradle_args, 'jacoco') }}
with:
file: ./build/reports/jacoco/jacocoReport/jacocoReport.xml
[parent: 841217a7f94b]