agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v3 1/5] Add regression tests for built-in encoding conversions. 17+ messages / 2 participants [nested] [flat]
* [PATCH v3 1/5] Add regression tests for built-in encoding conversions. @ 2021-02-01 15:10 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:10 UTC (permalink / raw) This doesn't cover every conversion, but it covers all the internal functions in conv.c that are used to implement the conversions. --- src/test/regress/expected/conversion.out | 493 +++++++++++++++++++++++ src/test/regress/sql/conversion.sql | 182 +++++++++ 2 files changed, 675 insertions(+) diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 62c10671685..38f8cef0f38 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -37,3 +37,496 @@ DROP CONVERSION mydef; -- RESET SESSION AUTHORIZATION; DROP USER regress_conversion_user; +-- +-- Test built-in conversion functions. +-- +-- helper function to test a conversion +create or replace function test_conv( + input IN bytea, + src_encoding IN text, + dst_encoding IN text, + result OUT bytea, + errorat OUT bytea, + error OUT text) +language plpgsql as +$$ +declare + validlen int; +begin + -- Try to perform the conversion. If it fails, catch the error and return + -- it to the caller. + begin + select * into result from convert(input, src_encoding, dst_encoding); + validlen = length(input); + errorat = NULL; + error := NULL; + exception when others then + result = NULL; + errorat = NULL; + error := sqlerrm; + end; + return; +end; +$$; +-- +-- UTF-8 +-- +CREATE TABLE utf8_inputs (inbytes bytea, description text); +insert into utf8_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\xc3a4c3b6', 'valid, extra latin chars'), + ('\xd184d0bed0be', 'valid, cyrillic'), + ('\x666f6fe8b1a1', 'valid, kanji/Chinese'), + ('\xe382abe3829a', 'valid, two chars that combine to one in EUC_JIS_2004'), + ('\xe382ab', 'only first half of combined char in EUC_JIS_2004'), + ('\xecbd94eb81bceba6ac', 'valid, Hangul, Korean'), + ('\x666f6fefa8aa', 'valid, needs mapping function to convert to GB18030'), + ('\x66e8b1ff6f6f', 'invalid byte sequence'), + ('\x66006f', 'invalid, NUL byte'), + ('\x666f6fe8b100', 'invalid, NUL byte'), + ('\x666f6fe8b1', 'incomplete character at end'); +-- Test UTF-8 verification +select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; + description | result | errorat | error +------------------------------------------------------+----------------------+---------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +-- Test conversions from UTF-8 +select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 +(12 rows) + +-- +-- EUC_JIS_2004 +-- +CREATE TABLE euc_jis_2004_inputs (inbytes bytea, description text); +insert into euc_jis_2004_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fbedd', 'valid'), + ('\xa5f7', 'valid, translates to two UTF-8 chars '), + ('\xbeddbe', 'incomplete char '), + ('\x666f6f00bedd', 'invalid, NUL byte'), + ('\x666f6fbe00dd', 'invalid, NUL byte'), + ('\x666f6fbedd00', 'invalid, NUL byte'), + ('\xbe04', 'invalid byte sequence'); +-- Test EUC_JIS_2004 verification +select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 +(8 rows) + +-- Test conversions from EUC_JIS_2004 +select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 +(8 rows) + +-- +-- SHIFT-JIS-2004 +-- +CREATE TABLE shiftjis2004_inputs (inbytes bytea, description text); +insert into shiftjis2004_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6f8fdb', 'valid'), + ('\x666f6f81c0', 'valid, no translation to UTF-8'), + ('\x666f6f82f5', 'valid, translates to two UTF-8 chars '), + ('\x666f6f8fdb8f', 'incomplete char '), + ('\x666f6f820a', 'incomplete char, followed by newline '), + ('\x666f6f008fdb', 'invalid, NUL byte'), + ('\x666f6f8f00db', 'invalid, NUL byte'), + ('\x666f6f8fdb00', 'invalid, NUL byte'); +-- Test SHIFT-JIS-2004 verification +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 +(9 rows) + +-- Test conversions from SHIFT-JIS-2004 +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 +(9 rows) + +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 +(9 rows) + +-- +-- GB18030 +-- +CREATE TABLE gb18030_inputs (inbytes bytea, description text); +insert into gb18030_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fcff3', 'valid'), + ('\x666f6f8431a530', 'valid, no translation to UTF-8'), + ('\x666f6f84309c38', 'valid, translates to UTF-8 by mapping function'), + ('\x666f6f84309c', 'incomplete char '), + ('\x666f6f84309c0a', 'incomplete char, followed by newline '), + ('\x666f6f84309c3800', 'invalid, NUL byte'), + ('\x666f6f84309c0038', 'invalid, NUL byte'); +-- Test GB18030 verification +select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 +(8 rows) + +-- Test conversions from GB18030 +select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 +(8 rows) + +-- +-- ISO-8859-5 +-- +CREATE TABLE iso8859_5_inputs (inbytes bytea, description text); +insert into iso8859_5_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\xe4dede', 'valid'), + ('\x00', 'invalid, NUL byte'), + ('\xe400dede', 'invalid, NUL byte'), + ('\xe4dede00', 'invalid, NUL byte'); +-- Test ISO-88591 verification +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; + description | inbytes | result | errorat | error +-------------------+------------+----------+---------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 +(5 rows) + +-- Test conversions from ISO-88591 +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; + description | inbytes | result | errorat | error +-------------------+------------+----------------+---------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 +(5 rows) + +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; + description | inbytes | result | errorat | error +-------------------+------------+----------+---------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 +(5 rows) + +select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; + description | inbytes | result | errorat | error +-------------------+------------+----------------+---------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 +(5 rows) + +-- +-- Big5 +-- +CREATE TABLE big5_inputs (inbytes bytea, description text); +insert into big5_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fb648', 'valid'), + ('\x666f6fa27f', 'valid, no translation to UTF-8'), + ('\x666f6fb60048', 'invalid, NUL byte'), + ('\x666f6fb64800', 'invalid, NUL byte'); +-- Test Big5 verification +select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+---------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 +(5 rows) + +-- Test conversions from Big5 +select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 +(5 rows) + +select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+---------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 +(5 rows) + +-- MULE_INTERNAL +CREATE TABLE mic_inputs (inbytes bytea, description text); +insert into mic_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x8bc68bcf8bcf', 'valid (in KOI8R)'), + ('\x8bc68bcf8b', 'invalid,incomplete char'), + ('\x92bedd', 'valid (in SHIFT_JIS)'), + ('\x92be', 'invalid, incomplete char)'), + ('\x666f6f95a3c1', 'valid (in Big5)'), + ('\x666f6f95a3', 'invalid, incomplete char'), + ('\x9200bedd', 'invalid, NUL byte'), + ('\x92bedd00', 'invalid, NUL byte'), + ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); +-- Test MULE_INTERNAL verification +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; + description | inbytes | result | errorat | error +---------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 +(10 rows) + +-- Test conversions from MULE_INTERNAL +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; + description | inbytes | result | errorat | error +---------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 +(10 rows) + +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; + description | inbytes | result | errorat | error +---------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 +(10 rows) + +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 +(10 rows) + +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; + description | inbytes | result | errorat | error +---------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 +(10 rows) + diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 02cf39f1ce9..644531d3333 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -34,3 +34,185 @@ DROP CONVERSION mydef; -- RESET SESSION AUTHORIZATION; DROP USER regress_conversion_user; + + + +-- +-- Test built-in conversion functions. +-- + +-- helper function to test a conversion +create or replace function test_conv( + input IN bytea, + src_encoding IN text, + dst_encoding IN text, + + result OUT bytea, + errorat OUT bytea, + error OUT text) +language plpgsql as +$$ +declare + validlen int; +begin + -- Try to perform the conversion. If it fails, catch the error and return + -- it to the caller. + begin + select * into result from convert(input, src_encoding, dst_encoding); + validlen = length(input); + errorat = NULL; + error := NULL; + exception when others then + result = NULL; + errorat = NULL; + error := sqlerrm; + end; + return; +end; +$$; + +-- +-- UTF-8 +-- +CREATE TABLE utf8_inputs (inbytes bytea, description text); +insert into utf8_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\xc3a4c3b6', 'valid, extra latin chars'), + ('\xd184d0bed0be', 'valid, cyrillic'), + ('\x666f6fe8b1a1', 'valid, kanji/Chinese'), + ('\xe382abe3829a', 'valid, two chars that combine to one in EUC_JIS_2004'), + ('\xe382ab', 'only first half of combined char in EUC_JIS_2004'), + ('\xecbd94eb81bceba6ac', 'valid, Hangul, Korean'), + ('\x666f6fefa8aa', 'valid, needs mapping function to convert to GB18030'), + ('\x66e8b1ff6f6f', 'invalid byte sequence'), + ('\x66006f', 'invalid, NUL byte'), + ('\x666f6fe8b100', 'invalid, NUL byte'), + ('\x666f6fe8b1', 'incomplete character at end'); + +-- Test UTF-8 verification +select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; +-- Test conversions from UTF-8 +select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; +select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; +select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; +select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; + +-- +-- EUC_JIS_2004 +-- +CREATE TABLE euc_jis_2004_inputs (inbytes bytea, description text); +insert into euc_jis_2004_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fbedd', 'valid'), + ('\xa5f7', 'valid, translates to two UTF-8 chars '), + ('\xbeddbe', 'incomplete char '), + ('\x666f6f00bedd', 'invalid, NUL byte'), + ('\x666f6fbe00dd', 'invalid, NUL byte'), + ('\x666f6fbedd00', 'invalid, NUL byte'), + ('\xbe04', 'invalid byte sequence'); + +-- Test EUC_JIS_2004 verification +select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; +-- Test conversions from EUC_JIS_2004 +select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; + +-- +-- SHIFT-JIS-2004 +-- +CREATE TABLE shiftjis2004_inputs (inbytes bytea, description text); +insert into shiftjis2004_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6f8fdb', 'valid'), + ('\x666f6f81c0', 'valid, no translation to UTF-8'), + ('\x666f6f82f5', 'valid, translates to two UTF-8 chars '), + ('\x666f6f8fdb8f', 'incomplete char '), + ('\x666f6f820a', 'incomplete char, followed by newline '), + ('\x666f6f008fdb', 'invalid, NUL byte'), + ('\x666f6f8f00db', 'invalid, NUL byte'), + ('\x666f6f8fdb00', 'invalid, NUL byte'); + +-- Test SHIFT-JIS-2004 verification +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; +-- Test conversions from SHIFT-JIS-2004 +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; +select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; + +-- +-- GB18030 +-- +CREATE TABLE gb18030_inputs (inbytes bytea, description text); +insert into gb18030_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fcff3', 'valid'), + ('\x666f6f8431a530', 'valid, no translation to UTF-8'), + ('\x666f6f84309c38', 'valid, translates to UTF-8 by mapping function'), + ('\x666f6f84309c', 'incomplete char '), + ('\x666f6f84309c0a', 'incomplete char, followed by newline '), + ('\x666f6f84309c3800', 'invalid, NUL byte'), + ('\x666f6f84309c0038', 'invalid, NUL byte'); + +-- Test GB18030 verification +select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; +-- Test conversions from GB18030 +select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; + + +-- +-- ISO-8859-5 +-- +CREATE TABLE iso8859_5_inputs (inbytes bytea, description text); +insert into iso8859_5_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\xe4dede', 'valid'), + ('\x00', 'invalid, NUL byte'), + ('\xe400dede', 'invalid, NUL byte'), + ('\xe4dede00', 'invalid, NUL byte'); + +-- Test ISO-88591 verification +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; +-- Test conversions from ISO-88591 +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; +select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; +select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; + +-- +-- Big5 +-- +CREATE TABLE big5_inputs (inbytes bytea, description text); +insert into big5_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x666f6fb648', 'valid'), + ('\x666f6fa27f', 'valid, no translation to UTF-8'), + ('\x666f6fb60048', 'invalid, NUL byte'), + ('\x666f6fb64800', 'invalid, NUL byte'); + +-- Test Big5 verification +select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; +-- Test conversions from Big5 +select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; +select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; + + +-- MULE_INTERNAL +CREATE TABLE mic_inputs (inbytes bytea, description text); +insert into mic_inputs values + ('\x666f6f', 'valid, pure ASCII'), + ('\x8bc68bcf8bcf', 'valid (in KOI8R)'), + ('\x8bc68bcf8b', 'invalid,incomplete char'), + ('\x92bedd', 'valid (in SHIFT_JIS)'), + ('\x92be', 'invalid, incomplete char)'), + ('\x666f6f95a3c1', 'valid (in Big5)'), + ('\x666f6f95a3', 'invalid, incomplete char'), + ('\x9200bedd', 'invalid, NUL byte'), + ('\x92bedd00', 'invalid, NUL byte'), + ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); + +-- Test MULE_INTERNAL verification +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; +-- Test conversions from MULE_INTERNAL +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; +select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0002-Change-conversion-function-signature.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v3-0002-Change-conversion-function-signature.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v3 01/13] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. ExecBitmapHeapInitializeWorker() overwrote the snapshot in the scan descriptor with the correct one provided by the parallel leader. Since ExecBitmapHeapInitializeWorker() is now called before the scan descriptor has been created, save the worker's snapshot in the BitmapHeapScanState and pass it to table_beginscan_bm(). --- src/backend/access/table/tableam.c | 11 ------ src/backend/executor/nodeBitmapHeapscan.c | 47 ++++++++++++++++++----- src/include/access/tableam.h | 10 ++--- src/include/nodes/execnodes.h | 2 + 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 6ed8cca05a1..e78d793f69c 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,17 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } -void -table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot) -{ - Assert(IsMVCCSnapshot(snapshot)); - - RegisterSnapshot(snapshot); - scan->rs_snapshot = snapshot; - scan->rs_flags |= SO_TEMP_SNAPSHOT; -} - - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index c1e81ebed63..44bf38be3c9 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -181,6 +181,34 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + Snapshot snapshot = node->ss.ps.state->es_snapshot; + uint32 extra_flags = 0; + + /* + * Parallel workers must use the snapshot initialized by the + * parallel leader. + */ + if (node->worker_snapshot) + { + snapshot = node->worker_snapshot; + extra_flags |= SO_TEMP_SNAPSHOT; + } + + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + snapshot, + 0, + NULL, + extra_flags); + } + node->initialized = true; } @@ -604,7 +632,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -681,7 +710,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -739,6 +770,7 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) */ scanstate->can_skip_fetch = (node->scan.plan.qual == NIL && node->scan.plan.targetlist == NIL); + scanstate->worker_snapshot = NULL; /* * Miscellaneous initialization @@ -787,11 +819,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ @@ -930,13 +957,13 @@ ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, ParallelWorkerContext *pwcxt) { ParallelBitmapHeapState *pstate; - Snapshot snapshot; Assert(node->ss.ps.state->es_query_dsa != NULL); pstate = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, false); node->pstate = pstate; - snapshot = RestoreSnapshot(pstate->phs_snapshot_data); - table_scan_update_snapshot(node->ss.ss_currentScanDesc, snapshot); + node->worker_snapshot = RestoreSnapshot(pstate->phs_snapshot_data); + Assert(IsMVCCSnapshot(node->worker_snapshot)); + RegisterSnapshot(node->worker_snapshot); } diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 5f8474871d2..5375dd7150f 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -944,9 +944,10 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(Relation rel, Snapshot snapshot, - int nkeys, struct ScanKeyData *key) + int nkeys, struct ScanKeyData *key, + uint32 extra_flags) { - uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE; + uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE | extra_flags; return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); } @@ -1038,11 +1039,6 @@ table_rescan_set_params(TableScanDesc scan, struct ScanKeyData *key, allow_pagemode); } -/* - * Update snapshot used by the scan. - */ -extern void table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot); - /* * Return next tuple from `scan`, store in slot. */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 444a5f0fd57..00c75fb10e2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1726,6 +1726,7 @@ typedef struct ParallelBitmapHeapState * shared_tbmiterator shared iterator * shared_prefetch_iterator shared iterator for prefetching * pstate shared state for parallel bitmap scan + * worker_snapshot snapshot for parallel worker * ---------------- */ typedef struct BitmapHeapScanState @@ -1750,6 +1751,7 @@ typedef struct BitmapHeapScanState TBMSharedIterator *shared_tbmiterator; TBMSharedIterator *shared_prefetch_iterator; ParallelBitmapHeapState *pstate; + Snapshot worker_snapshot; } BitmapHeapScanState; /* ---------------- -- 2.37.2 --fa5b4qdlc6zixc7z Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v3-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v4 01/14] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. ExecBitmapHeapInitializeWorker() overwrote the snapshot in the scan descriptor with the correct one provided by the parallel leader. Since ExecBitmapHeapInitializeWorker() is now called before the scan descriptor has been created, save the worker's snapshot in the BitmapHeapScanState and pass it to table_beginscan_bm(). --- src/backend/access/table/tableam.c | 11 ------ src/backend/executor/nodeBitmapHeapscan.c | 47 ++++++++++++++++++----- src/include/access/tableam.h | 10 ++--- src/include/nodes/execnodes.h | 2 + 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 6ed8cca05a1..e78d793f69c 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,17 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } -void -table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot) -{ - Assert(IsMVCCSnapshot(snapshot)); - - RegisterSnapshot(snapshot); - scan->rs_snapshot = snapshot; - scan->rs_flags |= SO_TEMP_SNAPSHOT; -} - - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index c1e81ebed63..44bf38be3c9 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -181,6 +181,34 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + Snapshot snapshot = node->ss.ps.state->es_snapshot; + uint32 extra_flags = 0; + + /* + * Parallel workers must use the snapshot initialized by the + * parallel leader. + */ + if (node->worker_snapshot) + { + snapshot = node->worker_snapshot; + extra_flags |= SO_TEMP_SNAPSHOT; + } + + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + snapshot, + 0, + NULL, + extra_flags); + } + node->initialized = true; } @@ -604,7 +632,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -681,7 +710,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -739,6 +770,7 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) */ scanstate->can_skip_fetch = (node->scan.plan.qual == NIL && node->scan.plan.targetlist == NIL); + scanstate->worker_snapshot = NULL; /* * Miscellaneous initialization @@ -787,11 +819,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ @@ -930,13 +957,13 @@ ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, ParallelWorkerContext *pwcxt) { ParallelBitmapHeapState *pstate; - Snapshot snapshot; Assert(node->ss.ps.state->es_query_dsa != NULL); pstate = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, false); node->pstate = pstate; - snapshot = RestoreSnapshot(pstate->phs_snapshot_data); - table_scan_update_snapshot(node->ss.ss_currentScanDesc, snapshot); + node->worker_snapshot = RestoreSnapshot(pstate->phs_snapshot_data); + Assert(IsMVCCSnapshot(node->worker_snapshot)); + RegisterSnapshot(node->worker_snapshot); } diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 5f8474871d2..5375dd7150f 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -944,9 +944,10 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(Relation rel, Snapshot snapshot, - int nkeys, struct ScanKeyData *key) + int nkeys, struct ScanKeyData *key, + uint32 extra_flags) { - uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE; + uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE | extra_flags; return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); } @@ -1038,11 +1039,6 @@ table_rescan_set_params(TableScanDesc scan, struct ScanKeyData *key, allow_pagemode); } -/* - * Update snapshot used by the scan. - */ -extern void table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot); - /* * Return next tuple from `scan`, store in slot. */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 444a5f0fd57..00c75fb10e2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1726,6 +1726,7 @@ typedef struct ParallelBitmapHeapState * shared_tbmiterator shared iterator * shared_prefetch_iterator shared iterator for prefetching * pstate shared state for parallel bitmap scan + * worker_snapshot snapshot for parallel worker * ---------------- */ typedef struct BitmapHeapScanState @@ -1750,6 +1751,7 @@ typedef struct BitmapHeapScanState TBMSharedIterator *shared_tbmiterator; TBMSharedIterator *shared_prefetch_iterator; ParallelBitmapHeapState *pstate; + Snapshot worker_snapshot; } BitmapHeapScanState; /* ---------------- -- 2.37.2 --5aaqsqqhbq27q3qo Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v4-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v5 01/14] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. ExecBitmapHeapInitializeWorker() overwrote the snapshot in the scan descriptor with the correct one provided by the parallel leader. Since ExecBitmapHeapInitializeWorker() is now called before the scan descriptor has been created, save the worker's snapshot in the BitmapHeapScanState and pass it to table_beginscan_bm(). --- src/backend/access/table/tableam.c | 11 ------ src/backend/executor/nodeBitmapHeapscan.c | 47 ++++++++++++++++++----- src/include/access/tableam.h | 10 ++--- src/include/nodes/execnodes.h | 2 + 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 6ed8cca05a1..e78d793f69c 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,17 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } -void -table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot) -{ - Assert(IsMVCCSnapshot(snapshot)); - - RegisterSnapshot(snapshot); - scan->rs_snapshot = snapshot; - scan->rs_flags |= SO_TEMP_SNAPSHOT; -} - - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index c1e81ebed63..44bf38be3c9 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -181,6 +181,34 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + Snapshot snapshot = node->ss.ps.state->es_snapshot; + uint32 extra_flags = 0; + + /* + * Parallel workers must use the snapshot initialized by the + * parallel leader. + */ + if (node->worker_snapshot) + { + snapshot = node->worker_snapshot; + extra_flags |= SO_TEMP_SNAPSHOT; + } + + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + snapshot, + 0, + NULL, + extra_flags); + } + node->initialized = true; } @@ -604,7 +632,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -681,7 +710,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -739,6 +770,7 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) */ scanstate->can_skip_fetch = (node->scan.plan.qual == NIL && node->scan.plan.targetlist == NIL); + scanstate->worker_snapshot = NULL; /* * Miscellaneous initialization @@ -787,11 +819,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ @@ -930,13 +957,13 @@ ExecBitmapHeapInitializeWorker(BitmapHeapScanState *node, ParallelWorkerContext *pwcxt) { ParallelBitmapHeapState *pstate; - Snapshot snapshot; Assert(node->ss.ps.state->es_query_dsa != NULL); pstate = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, false); node->pstate = pstate; - snapshot = RestoreSnapshot(pstate->phs_snapshot_data); - table_scan_update_snapshot(node->ss.ss_currentScanDesc, snapshot); + node->worker_snapshot = RestoreSnapshot(pstate->phs_snapshot_data); + Assert(IsMVCCSnapshot(node->worker_snapshot)); + RegisterSnapshot(node->worker_snapshot); } diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 5f8474871d2..5375dd7150f 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -944,9 +944,10 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(Relation rel, Snapshot snapshot, - int nkeys, struct ScanKeyData *key) + int nkeys, struct ScanKeyData *key, + uint32 extra_flags) { - uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE; + uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE | extra_flags; return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); } @@ -1038,11 +1039,6 @@ table_rescan_set_params(TableScanDesc scan, struct ScanKeyData *key, allow_pagemode); } -/* - * Update snapshot used by the scan. - */ -extern void table_scan_update_snapshot(TableScanDesc scan, Snapshot snapshot); - /* * Return next tuple from `scan`, store in slot. */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 444a5f0fd57..00c75fb10e2 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1726,6 +1726,7 @@ typedef struct ParallelBitmapHeapState * shared_tbmiterator shared iterator * shared_prefetch_iterator shared iterator for prefetching * pstate shared state for parallel bitmap scan + * worker_snapshot snapshot for parallel worker * ---------------- */ typedef struct BitmapHeapScanState @@ -1750,6 +1751,7 @@ typedef struct BitmapHeapScanState TBMSharedIterator *shared_tbmiterator; TBMSharedIterator *shared_prefetch_iterator; ParallelBitmapHeapState *pstate; + Snapshot worker_snapshot; } BitmapHeapScanState; /* ---------------- -- 2.37.2 --6kjpcnqyi64ibp5i Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v5-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v13 01/16] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 805d222ceb..b0f61c65f3 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..93fdcd226b 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --cuuqjeyokkhgd736 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v13-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v15 01/13] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) It makes more sense for BitmapHeapScan to scan the index, build the bitmap, and then begin the scan on the underlying table. This is primarily a cosmetic change for now, but later commits will pass parameters to table_beginscan_bm() that are unavailable in ExecInitBitmapHeapScan(). Author: Melanie Plageman Reviewed-by: Andres Freund, Heikki Linnakangas, Tomas Vondra Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/executor/nodeBitmapHeapscan.c | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..c8c466e3c5 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,21 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = table_beginscan_bm(node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + + node->ss.ss_currentScanDesc = scan; + } + node->initialized = true; } @@ -601,7 +616,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +694,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +801,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --pbix6fw3h4kvmjae Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v15-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v16 01/18] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) It makes more sense for BitmapHeapScan to scan the index, build the bitmap, and then begin the scan on the underlying table. This is primarily a cosmetic change for now, but later commits will pass parameters to table_beginscan_bm() that are unavailable in ExecInitBitmapHeapScan(). Author: Melanie Plageman Reviewed-by: Andres Freund, Heikki Linnakangas, Tomas Vondra Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/executor/nodeBitmapHeapscan.c | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..c8c466e3c5 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,21 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = table_beginscan_bm(node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + + node->ss.ss_currentScanDesc = scan; + } + node->initialized = true; } @@ -601,7 +616,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +694,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +801,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --e3xl7h75mzefinno Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v16-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v17 1/9] BitmapHeapScan: begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) It makes more sense for BitmapHeapScan to scan the index, build the bitmap, and only then begin the scan on the underlying table. This is mostly a cosmetic change for now, but later commits will need to pass parameters to table_beginscan_bm() that are unavailable in ExecInitBitmapHeapScan(). Author: Melanie Plageman Reviewed-by: Tomas Vondra, Andres Freund, Heikki Linnakangas Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/executor/nodeBitmapHeapscan.c | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aabe..c8c466e3c5c 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,21 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = table_beginscan_bm(node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + + node->ss.ss_currentScanDesc = scan; + } + node->initialized = true; } @@ -601,7 +616,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +694,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +801,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --etwsg7gfkf6ny266 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v17-0002-BitmapHeapScan-postpone-setting-can_skip_fetch.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v18 01/10] BitmapHeapScan: begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) It makes more sense for BitmapHeapScan to scan the index, build the bitmap, and only then begin the scan on the underlying table. This is mostly a cosmetic change for now, but later commits will need to pass parameters to table_beginscan_bm() that are unavailable in ExecInitBitmapHeapScan(). Author: Melanie Plageman Reviewed-by: Tomas Vondra, Andres Freund, Heikki Linnakangas Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/executor/nodeBitmapHeapscan.c | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aabe..c8c466e3c5c 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,21 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = table_beginscan_bm(node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + + node->ss.ss_currentScanDesc = scan; + } + node->initialized = true; } @@ -601,7 +616,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +694,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +801,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --zmcd37za4qx3kv73 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v18-0002-BitmapHeapScan-postpone-setting-can_skip_fetch.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v19 01/21] BitmapHeapScan: begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) It makes more sense for BitmapHeapScan to scan the index, build the bitmap, and only then begin the scan on the underlying table. This is mostly a cosmetic change for now, but later commits will need to pass parameters to table_beginscan_bm() that are unavailable in ExecInitBitmapHeapScan(). Author: Melanie Plageman Reviewed-by: Tomas Vondra, Andres Freund, Heikki Linnakangas Discussion: https://postgr.es/m/CAAKRu_ZwCwWFeL_H3ia26bP2e7HiKLWt0ZmGXPVwPO6uXq0vaA%40mail.gmail.com --- src/backend/executor/nodeBitmapHeapscan.c | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aabe..c8c466e3c5c 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,21 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = table_beginscan_bm(node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + + node->ss.ss_currentScanDesc = scan; + } + node->initialized = true; } @@ -601,7 +616,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +694,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +801,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --clv5reqgj4vdihqz Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v19-0002-BitmapHeapScan-postpone-setting-can_skip_fetch.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v6 01/14] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index e57a0b7ea31..e78d793f69c 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index ca548e44eb4..6d756001fdb 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --w4wcjcocxsm37usi Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v6-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v7 01/13] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index e57a0b7ea31..e78d793f69c 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aabe..93fdcd226bf 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --kqqpqghcwbcc3dt5 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v7-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v8 01/17] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index e57a0b7ea3..e78d793f69 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..93fdcd226b 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --xqq4defy3uncu6k6 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v8-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v9 01/17] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index e57a0b7ea3..e78d793f69 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..93fdcd226b 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --7mdtsjmrzitrgzgx Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v9-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v10 01/17] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index e57a0b7ea3..e78d793f69 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..93fdcd226b 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --3o7pc6dfau5a5hry Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v10-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v11 01/17] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 8d3675be95..a254e3175b 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aab..93fdcd226b 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --owzzsiozz6hgpp7e Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v11-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
* [PATCH v12 01/17] BitmapHeapScan begin scan after bitmap creation @ 2024-02-12 23:50 Melanie Plageman <[email protected]> 0 siblings, 0 replies; 17+ messages in thread From: Melanie Plageman @ 2024-02-12 23:50 UTC (permalink / raw) There is no reason for a BitmapHeapScan to begin the scan of the underlying table in ExecInitBitmapHeapScan(). Instead, do so after completing the index scan and building the bitmap. --- src/backend/access/table/tableam.c | 1 - src/backend/executor/nodeBitmapHeapscan.c | 26 +++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index 805d222cebc..b0f61c65f36 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -120,7 +120,6 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) NULL, flags); } - /* ---------------------------------------------------------------------------- * Parallel table scan related functions. * ---------------------------------------------------------------------------- diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index cee7f45aabe..93fdcd226bf 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -178,6 +178,20 @@ BitmapHeapNext(BitmapHeapScanState *node) } #endif /* USE_PREFETCH */ } + + /* + * If this is the first scan of the underlying table, create the table + * scan descriptor and begin the scan. + */ + if (!scan) + { + scan = node->ss.ss_currentScanDesc = table_beginscan_bm( + node->ss.ss_currentRelation, + node->ss.ps.state->es_snapshot, + 0, + NULL); + } + node->initialized = true; } @@ -601,7 +615,8 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) PlanState *outerPlan = outerPlanState(node); /* rescan to release any page pin */ - table_rescan(node->ss.ss_currentScanDesc, NULL); + if (node->ss.ss_currentScanDesc) + table_rescan(node->ss.ss_currentScanDesc, NULL); /* release bitmaps and buffers if any */ if (node->tbmiterator) @@ -678,7 +693,9 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * close heap scan */ - table_endscan(scanDesc); + if (scanDesc) + table_endscan(scanDesc); + } /* ---------------------------------------------------------------- @@ -783,11 +800,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ss_currentRelation = currentRelation; - scanstate->ss.ss_currentScanDesc = table_beginscan_bm(currentRelation, - estate->es_snapshot, - 0, - NULL); - /* * all done. */ -- 2.40.1 --6jpz2j246qmht4bt Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v12-0002-BitmapHeapScan-set-can_skip_fetch-later.patch" ^ permalink raw reply [nested|flat] 17+ messages in thread
end of thread, other threads:[~2024-02-12 23:50 UTC | newest] Thread overview: 17+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-01 15:10 [PATCH v3 1/5] Add regression tests for built-in encoding conversions. Heikki Linnakangas <[email protected]> 2024-02-12 23:50 [PATCH v11 01/17] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v5 01/14] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v9 01/17] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v3 01/13] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v16 01/18] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v19 01/21] BitmapHeapScan: begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v10 01/17] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v6 01/14] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v4 01/14] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v7 01/13] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v13 01/16] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v18 01/10] BitmapHeapScan: begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v8 01/17] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v12 01/17] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v15 01/13] BitmapHeapScan begin scan after bitmap creation Melanie Plageman <[email protected]> 2024-02-12 23:50 [PATCH v17 1/9] BitmapHeapScan: begin scan after bitmap creation Melanie Plageman <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox