public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. 27+ messages / 3 participants [nested] [flat]
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. @ 2021-02-01 15:28 Heikki Linnakangas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Heikki Linnakangas @ 2021-02-01 15:28 UTC (permalink / raw) TODO: When this is pushed, it probably makes more sense to add the whole regression as one commit after the code changes. But this is perhaps useful to keep as a separate commit for now. --- src/backend/utils/mb/mbutils.c | 55 ++ src/include/mb/pg_wchar.h | 6 + src/test/regress/expected/conversion.out | 575 +++++++++--------- .../regress/input/create_function_1.source | 4 + .../regress/output/create_function_1.source | 3 + src/test/regress/regress.c | 121 ++++ src/test/regress/sql/conversion.sql | 17 +- 7 files changed, 482 insertions(+), 299 deletions(-) diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 65753860e35..3e106027d75 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len, return result; } +/* + * Convert src string to another encoding. + * + * This function has a different API than the other conversion functions. + * The caller should've looked up the conversion function using + * FindDefaultConversionProc(). Unlike the other functions, the converted + * result is not palloc'd. It is written to a caller-supplied buffer instead. + * + * src_encoding - encoding to convert from + * dest_encoding - encoding to convert to + * src, srclen - input buffer and its length in bytes + * dest, destlen - destination buffer and its size in bytes + * + * The output is null-terminated. + * + * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output + * wouldn't necessarily fit in the output buffer, and the function will not + * convert the whole input. + * + * TODO: It would be nice to also return the number of bytes written to the + * caller, to avoid a call to strlen(). + */ +int +pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dest, int destlen, + bool noError) +{ + Datum result; + + /* + * If the destination buffer is not large enough to hold the + * result in the worst case, limit the input size passed to + * the conversion function. + * + * TODO: It would perhaps be more efficient to pass the destination + * buffer size to the conversion function, so that if the conversion + * expands less than the worst case, it could continue to fill up the + * whole buffer. + */ + if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH)) + srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH); + + result = OidFunctionCall6(proc, + Int32GetDatum(src_encoding), + Int32GetDatum(dest_encoding), + CStringGetDatum(src), + CStringGetDatum(dest), + Int32GetDatum(srclen), + BoolGetDatum(noError)); + return DatumGetInt32(result); +} + /* * Convert string to encoding encoding_name. The source * encoding is the DB encoding. diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 346a41a1f3d..9a22a6461d6 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname); extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len, int src_encoding, int dest_encoding); +extern int pg_do_encoding_conversion_buf(Oid proc, + int src_encoding, + int dest_encoding, + unsigned char *src, int srclen, + unsigned char *dst, int dstlen, + bool noError); extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out index 38f8cef0f38..571815683e9 100644 --- a/src/test/regress/expected/conversion.out +++ b/src/test/regress/expected/conversion.out @@ -53,24 +53,19 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -87,123 +82,123 @@ insert into utf8_inputs values ('\x666f6fe8b1', 'incomplete character at end'); -- Test UTF-8 verification select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs; - description | result | errorat | error -------------------------------------------------------+----------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | - valid, cyrillic | \xd184d0bed0be | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | - invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | result | errorat | error +------------------------------------------------------+----------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | | + valid, cyrillic | \xd184d0bed0be | | + valid, kanji/Chinese | \x666f6fe8b1a1 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | + invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- Test conversions from UTF-8 select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------+----------------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | - valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | | + valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | - valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, cyrillic | \xd184d0bed0be | \xc6cfcf | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R" + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R" + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs; - description | inbytes | result | errorat | error -------------------------------------------------------+----------------------+----------------------------+---------+----------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | - valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | - valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | - valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | - only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | - valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | - valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | - invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff - invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00 - invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 - incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 + description | inbytes | result | errorat | error +------------------------------------------------------+----------------------+----------------------------+--------------+----------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | | + valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | | + valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | | + valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | | + only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | | + valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | | + valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | | + invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff + invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00 + invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00 + incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 (12 rows) -- @@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values ('\xbe04', 'invalid byte sequence'); -- Test EUC_JIS_2004 verification select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fbedd | | - valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fbedd | | + valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | | + incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- Test conversions from EUC_JIS_2004 select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------+---------+-------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fbedd | \x666f6fe8b1a1 | | - valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | - incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe - invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 - invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 - invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------+----------+-------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fbedd | \x666f6fe8b1a1 | | + valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | | + incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe + invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00 + invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00 + invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04 (8 rows) -- @@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values ('\x666f6f8fdb00', 'invalid, NUL byte'); -- Test SHIFT-JIS-2004 verification select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6f8fdb | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6f8fdb | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | | + incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- Test conversions from SHIFT-JIS-2004 select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+----------------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+----------------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | | + incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs; - description | inbytes | result | errorat | error ----------------------------------------+----------------+--------------+---------+---------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6f8fdb | \x666f6fbedd | | - valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | - valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | - incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f - incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a - invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 - invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 - invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + description | inbytes | result | errorat | error +---------------------------------------+----------------+--------------+----------+---------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6f8fdb | \x666f6fbedd | | + valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | | + valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | | + incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f + incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a + invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 + invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00 + invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00 (9 rows) -- @@ -320,30 +315,30 @@ insert into gb18030_inputs values ('\x666f6f84309c0038', 'invalid, NUL byte'); -- Test GB18030 verification select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+------------------+---------+------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fcff3 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+------------------+--------------+------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fcff3 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | | + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- Test conversions from GB18030 select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs; - description | inbytes | result | errorat | error -------------------------------------------------+--------------------+----------------+---------+------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fcff3 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" - valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | - incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c - incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a - invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00 - invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 + description | inbytes | result | errorat | error +------------------------------------------------+--------------------+----------------+--------------+------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fcff3 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8" + valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | | + incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c + incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a + invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00 + invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00 (8 rows) -- @@ -358,44 +353,44 @@ insert into iso8859_5_inputs values ('\xe4dede00', 'invalid, NUL byte'); -- Test ISO-88591 verification select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xe4dede | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xe4dede | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- Test conversions from ISO-88591 select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xd184d0bed0be | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xd184d0bed0be | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \xc6cfcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \xc6cfcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs; - description | inbytes | result | errorat | error --------------------+------------+----------------+---------+------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \xe4dede | \x8bc68bcf8bcf | | - invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 - invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00 + description | inbytes | result | errorat | error +-------------------+------------+----------------+----------+------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \xe4dede | \x8bc68bcf8bcf | | + invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00 + invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00 (5 rows) -- @@ -410,37 +405,39 @@ insert into big5_inputs values ('\x666f6fb64800', 'invalid, NUL byte'); -- Test Big5 verification select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+--------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fb648 | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+--------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fb648 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) -- Test conversions from Big5 select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6fe8b1a1 | | - valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6fe8b1a1 | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8" + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - description | inbytes | result | errorat | error ---------------------------------+----------------+----------------+---------+------------------------------------------------------ - valid, pure ASCII | \x666f6f | \x666f6f | | - valid | \x666f6fb648 | \x666f6f95e2af | | - valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | - invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00 - invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00 + description | inbytes | result | errorat | error +--------------------------------+----------------+----------------+----------+------------------------------------------------------ + valid, pure ASCII | \x666f6f | \x666f6f | | + valid | \x666f6fb648 | \x666f6f95e2af | | + valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | | + invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00 + invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00 (5 rows) +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -455,78 +452,78 @@ insert into mic_inputs values ('\x8b00c68bcf8bcf', 'invalid, NUL byte'); -- Test MULE_INTERNAL verification select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | | + invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R" (10 rows) -- Test conversions from MULE_INTERNAL select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | | + invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5" (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + valid (in SHIFT_JIS) | \x92bedd | \x8fdb | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+--------------+---------+-------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+--------------+------------------+-------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | | + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5" + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs; - description | inbytes | result | errorat | error ----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------- - valid, pure ASCII | \x666f6f | \x666f6f | | - valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b - valid (in SHIFT_JIS) | \x92bedd | \xbedd | | - invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe - valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" - invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 - invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe - invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 - invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 + description | inbytes | result | errorat | error +---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------- + valid, pure ASCII | \x666f6f | \x666f6f | | + valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + valid (in SHIFT_JIS) | \x92bedd | \xbedd | | + invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe + valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP" + invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3 + invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe + invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00 + invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00 (10 rows) diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source index 412e339fcf2..6ba37fe63b6 100644 --- a/src/test/regress/input/create_function_1.source +++ b/src/test/regress/input/create_function_1.source @@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal) AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; + -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source index 4d78fa12289..cb38a039bf4 100644 --- a/src/test/regress/output/create_function_1.source +++ b/src/test/regress/output/create_function_1.source @@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal) RETURNS void AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func' LANGUAGE C; +CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion' + LANGUAGE C; -- Things that shouldn't work: CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL AS 'SELECT ''not an integer'';'; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 32ab9ed6b53..6dc43ff7fea 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -23,12 +23,15 @@ #include "access/htup_details.h" #include "access/transam.h" #include "access/xact.h" +#include "catalog/namespace.h" #include "catalog/pg_operator.h" #include "catalog/pg_type.h" #include "commands/sequence.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/spi.h" +#include "funcapi.h" +#include "mb/pg_wchar.h" #include "miscadmin.h" #include "nodes/supportnodes.h" #include "optimizer/optimizer.h" @@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } + + +PG_FUNCTION_INFO_V1(test_enc_conversion); +Datum +test_enc_conversion(PG_FUNCTION_ARGS) +{ + bytea *string = PG_GETARG_BYTEA_PP(0); + char *src_encoding_name = NameStr(*PG_GETARG_NAME(1)); + int src_encoding = pg_char_to_encoding(src_encoding_name); + char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2)); + int dest_encoding = pg_char_to_encoding(dest_encoding_name); + bool noError = PG_GETARG_BOOL(3); + TupleDesc tupdesc; + char *src; + char *dst; + bytea *retval; + Size srclen; + Size dstsize; + Oid proc; + int convertedbytes; + int dstlen; + Datum values[2]; + bool nulls[2]; + HeapTuple tuple; + + if (src_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid source encoding name \"%s\"", + src_encoding_name))); + if (dest_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid destination encoding name \"%s\"", + dest_encoding_name))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + tupdesc = BlessTupleDesc(tupdesc); + + srclen = VARSIZE_ANY_EXHDR(string); + src = VARDATA_ANY(string); + + if (src_encoding == dest_encoding) + { + /* just check that the source string is valid */ + int oklen; + + oklen = pg_encoding_verifymbstr(src_encoding, src, srclen); + + if (oklen == srclen) + { + convertedbytes = oklen; + retval = string; + } + else if (!noError) + { + report_invalid_encoding(src_encoding, src + oklen, srclen - oklen); + } + else + { + /* + * build bytea data type structure. + */ + Assert(oklen < srclen); + convertedbytes = oklen; + retval = (bytea *) palloc(oklen + VARHDRSZ); + SET_VARSIZE(retval, oklen + VARHDRSZ); + memcpy(VARDATA(retval), src, oklen); + } + } + else + { + proc = FindDefaultConversionProc(src_encoding, dest_encoding); + if (!OidIsValid(proc)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist", + pg_encoding_to_char(src_encoding), + pg_encoding_to_char(dest_encoding)))); + + if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH)) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("out of memory"), + errdetail("String of %d bytes is too long for encoding conversion.", + (int) srclen))); + + dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1; + dst = MemoryContextAlloc(CurrentMemoryContext, dstsize); + + /* perform conversion */ + convertedbytes = pg_do_encoding_conversion_buf(proc, + src_encoding, + dest_encoding, + (unsigned char *) src, srclen, + (unsigned char *) dst, dstsize, + noError); + dstlen = strlen(dst); + + /* + * build bytea data type structure. + */ + retval = (bytea *) palloc(dstlen + VARHDRSZ); + SET_VARSIZE(retval, dstlen + VARHDRSZ); + memcpy(VARDATA(retval), dst, dstlen); + + pfree(dst); + } + + MemSet(nulls, 0, sizeof(nulls)); + values[0] = Int32GetDatum(convertedbytes); + values[1] = PointerGetDatum(retval); + tuple = heap_form_tuple(tupdesc, values, nulls); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); +} diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql index 644531d3333..41e2686d46a 100644 --- a/src/test/regress/sql/conversion.sql +++ b/src/test/regress/sql/conversion.sql @@ -55,25 +55,21 @@ $$ declare validlen int; begin - -- Try to perform the conversion. If it fails, catch the error and return - -- it to the caller. begin - select * into result from convert(input, src_encoding, dst_encoding); - validlen = length(input); + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false); errorat = NULL; error := NULL; exception when others then - result = NULL; - errorat = NULL; + select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true); + errorat = substr(input, validlen + 1); error := sqlerrm; end; return; end; $$; --- --- UTF-8 --- +-- Test verification functions + CREATE TABLE utf8_inputs (inbytes bytea, description text); insert into utf8_inputs values ('\x666f6f', 'valid, pure ASCII'), @@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs; select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs; - +-- -- MULE_INTERNAL +-- CREATE TABLE mic_inputs (inbytes bytea, description text); insert into mic_inputs values ('\x666f6f', 'valid, pure ASCII'), -- 2.29.2 --------------39E46EF15EBFAA37D53332E5 Content-Type: text/x-patch; charset=UTF-8; name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Consider low startup cost in add_partial_path @ 2026-02-19 19:29 Robert Haas <[email protected]> 0 siblings, 1 reply; 27+ messages in thread From: Robert Haas @ 2026-02-19 19:29 UTC (permalink / raw) To: Richard Guo <[email protected]>; +Cc: Tomas Vondra <[email protected]>; James Coleman <[email protected]>; pgsql-hackers On Tue, Feb 17, 2026 at 2:44 AM Richard Guo <[email protected]> wrote: > Yeah, I agree that we should consider startup cost for partial paths, > given that cost_gather (and cost_gather_merge) accounts for the > subpath's startup_cost when calculating the cost of the Gather path. > > The changes LGTM. I noticed one comment that may need to be updated > accordingly: > > * As in add_path, the partial_pathlist is kept sorted with the cheapest > * total path in front. ... > > This is not true anymore. The partial_pathlist is now sorted by > disabled_nodes and then by cheapest total cost. Thanks. I have committed 0001 after adjusting that comment. Here's the remaining patch once again. No comments on this one? -- Robert Haas EDB: http://www.enterprisedb.com Attachments: [application/octet-stream] v56-0001-Consider-startup-cost-as-a-figure-of-merit-for-p.patch (17.1K, ../../CA+Tgmob57CDgacucdcDhW+=jNQnB6J4wx9b+WujKboDJM18wBw@mail.gmail.com/2-v56-0001-Consider-startup-cost-as-a-figure-of-merit-for-p.patch) download | inline diff: From bc3df418deb52a94dfe5f3c6da0b28480d908da0 Mon Sep 17 00:00:00 2001 From: Robert Haas <[email protected]> Date: Thu, 19 Feb 2026 14:02:16 -0500 Subject: [PATCH v56] Consider startup cost as a figure of merit for partial paths. Previously, the coments stated that there was no purpose to considering startup cost for partial paths, but this is not the case: it's perfectly reasonable to want a fast-start path for a plan that involves a LIMIT (perhaps over an aggregate, so that there is enough data being processed to justify parallel query but yet we don't want all the result rows). Accordingly, rewrite add_partial_path and add_partial_path_precheck to consider startup costs. This also fixes an independent bug in add_partial_path_precheck: commit e22253467942fdb100087787c3e1e3a8620c54b2 failed to update it to do anything with the new disabled_nodes field. That bug fix is formally separate from the rest of this patch and could be committed separately, but I think it makes more sense to fix both issues together, because then we can (as this commit does) just make add_partial_path_precheck do the cost comparisons in the same way as compare_path_costs_fuzzily, which hopefully reduces the chances of ending up with something that's still incorrect. This patch is based on earlier work on this topic by Tomas Vondra, but I have rewritten a great deal of it. Co-authored-by: Robert Haas <[email protected]> Co-authored-by: Tomas Vondra <[email protected]> --- src/backend/optimizer/path/joinpath.c | 3 + src/backend/optimizer/util/pathnode.c | 159 +++++++++++------- src/include/optimizer/pathnode.h | 2 +- .../regress/expected/incremental_sort.out | 18 +- src/test/regress/expected/join_hash.out | 13 +- src/test/regress/sql/join_hash.sql | 10 +- 6 files changed, 122 insertions(+), 83 deletions(-) diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index e0c00e26dd5..044560da7bf 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -1048,6 +1048,7 @@ try_partial_nestloop_path(PlannerInfo *root, initial_cost_nestloop(root, &workspace, jointype, nestloop_subtype, outer_path, inner_path, extra); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, pathkeys)) return; @@ -1237,6 +1238,7 @@ try_partial_mergejoin_path(PlannerInfo *root, extra); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, pathkeys)) return; @@ -1369,6 +1371,7 @@ try_partial_hashjoin_path(PlannerInfo *root, initial_cost_hashjoin(root, &workspace, jointype, hashclauses, outer_path, inner_path, extra, parallel_hash); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, NIL)) return; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index ef8ef6e89d3..7dc9288575f 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -778,10 +778,9 @@ add_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * * Because we don't consider parameterized paths here, we also don't * need to consider the row counts as a measure of quality: every path will - * produce the same number of rows. Neither do we need to consider startup - * costs: parallelism is only used for plans that will be run to completion. - * Therefore, this routine is much simpler than add_path: it needs to - * consider only disabled nodes, pathkeys and total cost. + * produce the same number of rows. However, we do need to consider the + * startup costs: this partial path could be used beneath a Limit node, + * so a fast-start plan could be correct. * * As with add_path, we pfree paths that are found to be dominated by * another partial path; this requires that there be no other references to @@ -819,52 +818,36 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) /* Compare pathkeys. */ keyscmp = compare_pathkeys(new_path->pathkeys, old_path->pathkeys); - /* Unless pathkeys are incompatible, keep just one of the two paths. */ + /* + * Unless pathkeys are incompatible, see if one of the paths dominates + * the other (both in startup and total cost). It may happen that one + * path has lower startup cost, the other has lower total cost. + */ if (keyscmp != PATHKEYS_DIFFERENT) { - if (unlikely(new_path->disabled_nodes != old_path->disabled_nodes)) + PathCostComparison costcmp; + + /* + * Do a fuzzy cost comparison with standard fuzziness limit. + */ + costcmp = compare_path_costs_fuzzily(new_path, old_path, + STD_FUZZ_FACTOR); + if (costcmp == COSTS_BETTER1) { - if (new_path->disabled_nodes > old_path->disabled_nodes) - accept_new = false; - else + if (keyscmp == PATHKEYS_BETTER1) remove_old = true; } - else if (new_path->total_cost > old_path->total_cost - * STD_FUZZ_FACTOR) + else if (costcmp == COSTS_BETTER2) { - /* New path costs more; keep it only if pathkeys are better. */ - if (keyscmp != PATHKEYS_BETTER1) + if (keyscmp == PATHKEYS_BETTER2) accept_new = false; } - else if (old_path->total_cost > new_path->total_cost - * STD_FUZZ_FACTOR) + else if (costcmp == COSTS_EQUAL) { - /* Old path costs more; keep it only if pathkeys are better. */ - if (keyscmp != PATHKEYS_BETTER2) + if (keyscmp == PATHKEYS_BETTER1) remove_old = true; - } - else if (keyscmp == PATHKEYS_BETTER1) - { - /* Costs are about the same, new path has better pathkeys. */ - remove_old = true; - } - else if (keyscmp == PATHKEYS_BETTER2) - { - /* Costs are about the same, old path has better pathkeys. */ - accept_new = false; - } - else if (old_path->total_cost > new_path->total_cost * 1.0000000001) - { - /* Pathkeys are the same, and the old path costs more. */ - remove_old = true; - } - else - { - /* - * Pathkeys are the same, and new path isn't materially - * cheaper. - */ - accept_new = false; + else if (keyscmp == PATHKEYS_BETTER2) + accept_new = false; } } @@ -915,16 +898,16 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) * add_partial_path_precheck * Check whether a proposed new partial path could possibly get accepted. * - * Unlike add_path_precheck, we can ignore startup cost and parameterization, - * since they don't matter for partial paths (see add_partial_path). But - * we do want to make sure we don't add a partial path if there's already - * a complete path that dominates it, since in that case the proposed path - * is surely a loser. + * Unlike add_path_precheck, we can ignore parameterization, since it doesn't + * matter for partial paths (see add_partial_path). But we do want to make + * sure we don't add a partial path if there's already a complete path that + * dominates it, since in that case the proposed path is surely a loser. */ bool add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, - Cost total_cost, List *pathkeys) + Cost startup_cost, Cost total_cost, List *pathkeys) { + bool consider_startup = parent_rel->consider_startup; ListCell *p1; /* @@ -934,25 +917,80 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * is clearly superior to some existing partial path -- at least, modulo * final cost computations. If so, we definitely want to consider it. * - * Unlike add_path(), we always compare pathkeys here. This is because we - * expect partial_pathlist to be very short, and getting a definitive + * Unlike add_path(), we never try to exit this loop early. This is because + * we expect partial_pathlist to be very short, and getting a definitive * answer at this stage avoids the need to call add_path_precheck. */ foreach(p1, parent_rel->partial_pathlist) { Path *old_path = (Path *) lfirst(p1); + PathCostComparison costcmp; PathKeysComparison keyscmp; - keyscmp = compare_pathkeys(pathkeys, old_path->pathkeys); - if (keyscmp != PATHKEYS_DIFFERENT) + /* + * First, compare costs and disabled nodes. This logic should be + * identical to compare_path_costs_fuzzily, except that one of the + * paths hasn't been created yet, and the fuzz factor is always + * STD_FUZZ_FACTOR. + */ + if (unlikely(old_path->disabled_nodes != disabled_nodes)) { - if (total_cost > old_path->total_cost * STD_FUZZ_FACTOR && - keyscmp != PATHKEYS_BETTER1) - return false; - if (old_path->total_cost > total_cost * STD_FUZZ_FACTOR && - keyscmp != PATHKEYS_BETTER2) - return true; + if (disabled_nodes < old_path->disabled_nodes) + costcmp = COSTS_BETTER1; + else + costcmp = COSTS_BETTER2; + } + else if (total_cost > old_path->total_cost * STD_FUZZ_FACTOR) + { + if (consider_startup && + old_path->startup_cost > startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_DIFFERENT; + else + costcmp = COSTS_BETTER2; + } + else if (old_path->total_cost > total_cost * STD_FUZZ_FACTOR) + { + if (consider_startup && + startup_cost > old_path->startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_DIFFERENT; + else + costcmp = COSTS_BETTER1; } + else if (startup_cost > old_path->startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_BETTER2; + else if (old_path->startup_cost > startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_BETTER1; + else + costcmp = COSTS_EQUAL; + + /* + * If one path wins on startup cost and the other on total cost, we + * can't say for sure which is better. + */ + if (costcmp == COSTS_DIFFERENT) + continue; + + /* + * If the two paths have different pathkeys, we can't say for sure + * which is better. + */ + keyscmp = compare_pathkeys(pathkeys, old_path->pathkeys); + if (keyscmp == PATHKEYS_DIFFERENT) + continue; + + /* + * If the existing path is cheaper and the pathkeys are equal or worse, + * the new path is not interesting. + */ + if (costcmp == COSTS_BETTER2 && keyscmp != PATHKEYS_BETTER1) + return false; + + /* + * If the new path is cheaper and the pathkeys are equal or better, + * it is definitely interesting. + */ + if (costcmp == COSTS_BETTER1 && keyscmp != PATHKEYS_BETTER2) + return true; } /* @@ -960,14 +998,9 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * clearly good enough that it might replace one. Compare it to * non-parallel plans. If it loses even before accounting for the cost of * the Gather node, we should definitely reject it. - * - * Note that we pass the total_cost to add_path_precheck twice. This is - * because it's never advantageous to consider the startup cost of a - * partial path; the resulting plans, if run in parallel, will be run to - * completion. */ - if (!add_path_precheck(parent_rel, disabled_nodes, total_cost, total_cost, - pathkeys, NULL)) + if (!add_path_precheck(parent_rel, disabled_nodes, startup_cost, + total_cost, pathkeys, NULL)) return false; return true; diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index cf8a654fa53..938510400cc 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -55,7 +55,7 @@ extern bool add_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, List *pathkeys, Relids required_outer); extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, - int disabled_nodes, + int disabled_nodes, Cost startup_cost, Cost total_cost, List *pathkeys); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out index fdec5b9ba52..29090dca1ba 100644 --- a/src/test/regress/expected/incremental_sort.out +++ b/src/test/regress/expected/incremental_sort.out @@ -1450,21 +1450,23 @@ explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1 set enable_incremental_sort = on; explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; - QUERY PLAN ----------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------- Limit -> Incremental Sort Sort Key: a, b, (sum(c)) Presorted Key: a, b - -> GroupAggregate + -> Finalize GroupAggregate Group Key: a, b -> Gather Merge Workers Planned: 2 - -> Incremental Sort - Sort Key: a, b - Presorted Key: a - -> Parallel Index Scan using t_a_idx on t -(12 rows) + -> Partial GroupAggregate + Group Key: a, b + -> Incremental Sort + Sort Key: a, b + Presorted Key: a + -> Parallel Index Scan using t_a_idx on t +(14 rows) -- Incremental sort vs. set operations with varno 0 set enable_hashagg to off; diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 4749f6ed70d..bc7cc76467f 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -76,8 +76,8 @@ insert into extremely_skewed update pg_class set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192 where relname = 'extremely_skewed'; --- Make a relation with a couple of enormous tuples. -create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +-- Make a relation with several enormous tuples. +create table wide as select generate_series(1, 3) as id, rpad('', 320000, 'x') as t; alter table wide set (parallel_workers = 2); -- The "optimal" case: the hash table fits in memory; we plan for 1 -- batch, we stick to that number, and peak memory usage stays within @@ -922,7 +922,7 @@ set work_mem = '128kB'; set hash_mem_multiplier = 1.0; explain (costs off) select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); QUERY PLAN ---------------------------------------------------------------- Finalize Aggregate @@ -934,10 +934,11 @@ explain (costs off) -> Parallel Seq Scan on wide -> Parallel Hash -> Parallel Seq Scan on wide wide_1 -(9 rows) + Filter: (id < 3) +(10 rows) select length(max(s.t)) -from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); +from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); length -------- 320000 @@ -947,7 +948,7 @@ select final > 1 as multibatch from hash_join_batches( $$ select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); $$); multibatch ------------ diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 49d3fd61856..53db1754bb2 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -83,8 +83,8 @@ update pg_class set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192 where relname = 'extremely_skewed'; --- Make a relation with a couple of enormous tuples. -create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +-- Make a relation with several enormous tuples. +create table wide as select generate_series(1, 3) as id, rpad('', 320000, 'x') as t; alter table wide set (parallel_workers = 2); -- The "optimal" case: the hash table fits in memory; we plan for 1 @@ -496,14 +496,14 @@ set work_mem = '128kB'; set hash_mem_multiplier = 1.0; explain (costs off) select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); select length(max(s.t)) -from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); +from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); select final > 1 as multibatch from hash_join_batches( $$ select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); $$); rollback to settings; -- 2.51.0 ^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Consider low startup cost in add_partial_path @ 2026-02-25 06:35 Richard Guo <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 1 reply; 27+ messages in thread From: Richard Guo @ 2026-02-25 06:35 UTC (permalink / raw) To: Robert Haas <[email protected]>; +Cc: Tomas Vondra <[email protected]>; James Coleman <[email protected]>; pgsql-hackers On Fri, Feb 20, 2026 at 4:30 AM Robert Haas <[email protected]> wrote: > Thanks. I have committed 0001 after adjusting that comment. > > Here's the remaining patch once again. No comments on this one? (Sorry I somehow lost track of this thread.) I noticed that in this patch, add_partial_path_precheck duplicates the entire logic of compare_path_costs_fuzzily. I wonder if it would be better to avoid this code duplication. I'm thinking that maybe we could extract the core math of compare_path_costs_fuzzily into an internal helper function that operates on cost values (and disabled_nodes) rather than on Path structures. We could then call this helper directly from within add_partial_path_precheck. That said, I'm not quite sure if that refactoring is worth the churn, or if there is a better way to do that. Otherwise, the patch looks good to me. - Richard ^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Consider low startup cost in add_partial_path @ 2026-03-04 22:04 Robert Haas <[email protected]> parent: Richard Guo <[email protected]> 0 siblings, 1 reply; 27+ messages in thread From: Robert Haas @ 2026-03-04 22:04 UTC (permalink / raw) To: Richard Guo <[email protected]>; +Cc: Tomas Vondra <[email protected]>; James Coleman <[email protected]>; pgsql-hackers On Wed, Feb 25, 2026 at 1:35 AM Richard Guo <[email protected]> wrote: > I noticed that in this patch, add_partial_path_precheck duplicates the > entire logic of compare_path_costs_fuzzily. I wonder if it would be > better to avoid this code duplication. > > I'm thinking that maybe we could extract the core math of > compare_path_costs_fuzzily into an internal helper function that > operates on cost values (and disabled_nodes) rather than on Path > structures. We could then call this helper directly from within > add_partial_path_precheck. Maybe, but it would be a ten-parameter function. > Otherwise, the patch looks good to me. Turns out that version had a few problems: 1. It didn't handle the case where costcmp was COSTS_BETTER{1|2} but keyscmp was PATHKEYS_EQUAL correctly. It had logic like this: if (costcmp == COSTS_BETTER1) { if (keyscmp == PATHKEYS_BETTER1) remove_old = true; }. This says that if the new path is better on cost and the new path is also better on pathkeys, throw away the old path. That's too weak. It should instead say if (costcmp == COSTS_BETTER1) { if (keyscmp != PATHKEYS_BETTER2) remove_old = true; }. That is, if the new path is better cost and is EQUAL OR BETTER on pathkeys, throw away the old path. As a result, v56 would have kept too many paths. 2. It failed to preserve an equivalent of the (old_path->total_cost > new_path->total_cost * 1.0000000001) test. It now tests compare_path_costs_fuzzily(new_path, old_path, 1.0000000001) == COSTS_BETTER1). 3. It wasn't pgindent-clean. Fixing the first of these problems results in one additional regression test change. The query is: explain (costs off) select count(*) from tenk1 t1 join tenk1 t2 on t1.unique1 = t2.unique2 join tenk1 t3 on t2.unique1 = t3.unique1 order by count(*); The sides of the t1-t2 join get swapped: the driving table is chosen to be t2 rather than t1. Without that fix, we keep both hash_join(t1,t2) and hash_join(t2,t1), The former has a total cost that is lower by a very small amount and therefore ends up first in the partial pathlist and so wins. With that fix, we see that hash_join(t2,t1) has an essentially equal total cost and a much lower startup cost, so we keep only that one. In other words, this change seems to be correct. Here's v57. -- Robert Haas EDB: http://www.enterprisedb.com Attachments: [application/octet-stream] v57-0001-Consider-startup-cost-as-a-figure-of-merit-for-p.patch (18.8K, ../../CA+TgmoZc9FJnZNWH5=mAbBGh6wOTZVhC_sQzpQn6KvFcRM832g@mail.gmail.com/2-v57-0001-Consider-startup-cost-as-a-figure-of-merit-for-p.patch) download | inline diff: From 5665d6ccae7a5af980053658c5416c2945a19290 Mon Sep 17 00:00:00 2001 From: Robert Haas <[email protected]> Date: Wed, 4 Mar 2026 17:03:35 -0500 Subject: [PATCH v57] Consider startup cost as a figure of merit for partial paths. Previously, the comments stated that there was no purpose to considering startup cost for partial paths, but this is not the case: it's perfectly reasonable to want a fast-start path for a plan that involves a LIMIT (perhaps over an aggregate, so that there is enough data being processed to justify parallel query but yet we don't want all the result rows). Accordingly, rewrite add_partial_path and add_partial_path_precheck to consider startup costs. This also fixes an independent bug in add_partial_path_precheck: commit e22253467942fdb100087787c3e1e3a8620c54b2 failed to update it to do anything with the new disabled_nodes field. That bug fix is formally separate from the rest of this patch and could be committed separately, but I think it makes more sense to fix both issues together, because then we can (as this commit does) just make add_partial_path_precheck do the cost comparisons in the same way as compare_path_costs_fuzzily, which hopefully reduces the chances of ending up with something that's still incorrect. This patch is based on earlier work on this topic by Tomas Vondra, but I have rewritten a great deal of it. Co-authored-by: Robert Haas <[email protected]> Co-authored-by: Tomas Vondra <[email protected]> --- src/backend/optimizer/path/joinpath.c | 3 + src/backend/optimizer/util/pathnode.c | 165 +++++++++++------- src/include/optimizer/pathnode.h | 2 +- .../regress/expected/incremental_sort.out | 28 +-- src/test/regress/expected/join_hash.out | 13 +- src/test/regress/sql/join_hash.sql | 10 +- 6 files changed, 133 insertions(+), 88 deletions(-) diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c index e0c00e26dd5..044560da7bf 100644 --- a/src/backend/optimizer/path/joinpath.c +++ b/src/backend/optimizer/path/joinpath.c @@ -1048,6 +1048,7 @@ try_partial_nestloop_path(PlannerInfo *root, initial_cost_nestloop(root, &workspace, jointype, nestloop_subtype, outer_path, inner_path, extra); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, pathkeys)) return; @@ -1237,6 +1238,7 @@ try_partial_mergejoin_path(PlannerInfo *root, extra); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, pathkeys)) return; @@ -1369,6 +1371,7 @@ try_partial_hashjoin_path(PlannerInfo *root, initial_cost_hashjoin(root, &workspace, jointype, hashclauses, outer_path, inner_path, extra, parallel_hash); if (!add_partial_path_precheck(joinrel, workspace.disabled_nodes, + workspace.startup_cost, workspace.total_cost, NIL)) return; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index ef8ef6e89d3..c94e077000f 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -778,10 +778,9 @@ add_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * * Because we don't consider parameterized paths here, we also don't * need to consider the row counts as a measure of quality: every path will - * produce the same number of rows. Neither do we need to consider startup - * costs: parallelism is only used for plans that will be run to completion. - * Therefore, this routine is much simpler than add_path: it needs to - * consider only disabled nodes, pathkeys and total cost. + * produce the same number of rows. However, we do need to consider the + * startup costs: this partial path could be used beneath a Limit node, + * so a fast-start plan could be correct. * * As with add_path, we pfree paths that are found to be dominated by * another partial path; this requires that there be no other references to @@ -819,52 +818,41 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) /* Compare pathkeys. */ keyscmp = compare_pathkeys(new_path->pathkeys, old_path->pathkeys); - /* Unless pathkeys are incompatible, keep just one of the two paths. */ + /* + * Unless pathkeys are incompatible, see if one of the paths dominates + * the other (both in startup and total cost). It may happen that one + * path has lower startup cost, the other has lower total cost. + */ if (keyscmp != PATHKEYS_DIFFERENT) { - if (unlikely(new_path->disabled_nodes != old_path->disabled_nodes)) + PathCostComparison costcmp; + + /* + * Do a fuzzy cost comparison with standard fuzziness limit. + */ + costcmp = compare_path_costs_fuzzily(new_path, old_path, + STD_FUZZ_FACTOR); + if (costcmp == COSTS_BETTER1) { - if (new_path->disabled_nodes > old_path->disabled_nodes) - accept_new = false; - else + if (keyscmp != PATHKEYS_BETTER2) remove_old = true; } - else if (new_path->total_cost > old_path->total_cost - * STD_FUZZ_FACTOR) + else if (costcmp == COSTS_BETTER2) { - /* New path costs more; keep it only if pathkeys are better. */ if (keyscmp != PATHKEYS_BETTER1) accept_new = false; } - else if (old_path->total_cost > new_path->total_cost - * STD_FUZZ_FACTOR) + else if (costcmp == COSTS_EQUAL) { - /* Old path costs more; keep it only if pathkeys are better. */ - if (keyscmp != PATHKEYS_BETTER2) + if (keyscmp == PATHKEYS_BETTER1) remove_old = true; - } - else if (keyscmp == PATHKEYS_BETTER1) - { - /* Costs are about the same, new path has better pathkeys. */ - remove_old = true; - } - else if (keyscmp == PATHKEYS_BETTER2) - { - /* Costs are about the same, old path has better pathkeys. */ - accept_new = false; - } - else if (old_path->total_cost > new_path->total_cost * 1.0000000001) - { - /* Pathkeys are the same, and the old path costs more. */ - remove_old = true; - } - else - { - /* - * Pathkeys are the same, and new path isn't materially - * cheaper. - */ - accept_new = false; + else if (keyscmp == PATHKEYS_BETTER2) + accept_new = false; + else if (compare_path_costs_fuzzily(new_path, old_path, + 1.0000000001) == COSTS_BETTER1) + remove_old = true; + else + accept_new = false; } } @@ -915,16 +903,16 @@ add_partial_path(RelOptInfo *parent_rel, Path *new_path) * add_partial_path_precheck * Check whether a proposed new partial path could possibly get accepted. * - * Unlike add_path_precheck, we can ignore startup cost and parameterization, - * since they don't matter for partial paths (see add_partial_path). But - * we do want to make sure we don't add a partial path if there's already - * a complete path that dominates it, since in that case the proposed path - * is surely a loser. + * Unlike add_path_precheck, we can ignore parameterization, since it doesn't + * matter for partial paths (see add_partial_path). But we do want to make + * sure we don't add a partial path if there's already a complete path that + * dominates it, since in that case the proposed path is surely a loser. */ bool add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, - Cost total_cost, List *pathkeys) + Cost startup_cost, Cost total_cost, List *pathkeys) { + bool consider_startup = parent_rel->consider_startup; ListCell *p1; /* @@ -934,25 +922,81 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * is clearly superior to some existing partial path -- at least, modulo * final cost computations. If so, we definitely want to consider it. * - * Unlike add_path(), we always compare pathkeys here. This is because we - * expect partial_pathlist to be very short, and getting a definitive - * answer at this stage avoids the need to call add_path_precheck. + * Unlike add_path(), we never try to exit this loop early. This is + * because we expect partial_pathlist to be very short, and getting a + * definitive answer at this stage avoids the need to call + * add_path_precheck. */ foreach(p1, parent_rel->partial_pathlist) { Path *old_path = (Path *) lfirst(p1); + PathCostComparison costcmp; PathKeysComparison keyscmp; - keyscmp = compare_pathkeys(pathkeys, old_path->pathkeys); - if (keyscmp != PATHKEYS_DIFFERENT) + /* + * First, compare costs and disabled nodes. This logic should be + * identical to compare_path_costs_fuzzily, except that one of the + * paths hasn't been created yet, and the fuzz factor is always + * STD_FUZZ_FACTOR. + */ + if (unlikely(old_path->disabled_nodes != disabled_nodes)) + { + if (disabled_nodes < old_path->disabled_nodes) + costcmp = COSTS_BETTER1; + else + costcmp = COSTS_BETTER2; + } + else if (total_cost > old_path->total_cost * STD_FUZZ_FACTOR) { - if (total_cost > old_path->total_cost * STD_FUZZ_FACTOR && - keyscmp != PATHKEYS_BETTER1) - return false; - if (old_path->total_cost > total_cost * STD_FUZZ_FACTOR && - keyscmp != PATHKEYS_BETTER2) - return true; + if (consider_startup && + old_path->startup_cost > startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_DIFFERENT; + else + costcmp = COSTS_BETTER2; + } + else if (old_path->total_cost > total_cost * STD_FUZZ_FACTOR) + { + if (consider_startup && + startup_cost > old_path->startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_DIFFERENT; + else + costcmp = COSTS_BETTER1; } + else if (startup_cost > old_path->startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_BETTER2; + else if (old_path->startup_cost > startup_cost * STD_FUZZ_FACTOR) + costcmp = COSTS_BETTER1; + else + costcmp = COSTS_EQUAL; + + /* + * If one path wins on startup cost and the other on total cost, we + * can't say for sure which is better. + */ + if (costcmp == COSTS_DIFFERENT) + continue; + + /* + * If the two paths have different pathkeys, we can't say for sure + * which is better. + */ + keyscmp = compare_pathkeys(pathkeys, old_path->pathkeys); + if (keyscmp == PATHKEYS_DIFFERENT) + continue; + + /* + * If the existing path is cheaper and the pathkeys are equal or + * worse, the new path is not interesting. + */ + if (costcmp == COSTS_BETTER2 && keyscmp != PATHKEYS_BETTER1) + return false; + + /* + * If the new path is cheaper and the pathkeys are equal or better, it + * is definitely interesting. + */ + if (costcmp == COSTS_BETTER1 && keyscmp != PATHKEYS_BETTER2) + return true; } /* @@ -960,14 +1004,9 @@ add_partial_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, * clearly good enough that it might replace one. Compare it to * non-parallel plans. If it loses even before accounting for the cost of * the Gather node, we should definitely reject it. - * - * Note that we pass the total_cost to add_path_precheck twice. This is - * because it's never advantageous to consider the startup cost of a - * partial path; the resulting plans, if run in parallel, will be run to - * completion. */ - if (!add_path_precheck(parent_rel, disabled_nodes, total_cost, total_cost, - pathkeys, NULL)) + if (!add_path_precheck(parent_rel, disabled_nodes, startup_cost, + total_cost, pathkeys, NULL)) return false; return true; diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index cf8a654fa53..938510400cc 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -55,7 +55,7 @@ extern bool add_path_precheck(RelOptInfo *parent_rel, int disabled_nodes, List *pathkeys, Relids required_outer); extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, - int disabled_nodes, + int disabled_nodes, Cost startup_cost, Cost total_cost, List *pathkeys); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, diff --git a/src/test/regress/expected/incremental_sort.out b/src/test/regress/expected/incremental_sort.out index fdec5b9ba52..1e6e020fea8 100644 --- a/src/test/regress/expected/incremental_sort.out +++ b/src/test/regress/expected/incremental_sort.out @@ -1450,21 +1450,23 @@ explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1 set enable_incremental_sort = on; explain (costs off) select a,b,sum(c) from t group by 1,2 order by 1,2,3 limit 1; - QUERY PLAN ----------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------- Limit -> Incremental Sort Sort Key: a, b, (sum(c)) Presorted Key: a, b - -> GroupAggregate + -> Finalize GroupAggregate Group Key: a, b -> Gather Merge Workers Planned: 2 - -> Incremental Sort - Sort Key: a, b - Presorted Key: a - -> Parallel Index Scan using t_a_idx on t -(12 rows) + -> Partial GroupAggregate + Group Key: a, b + -> Incremental Sort + Sort Key: a, b + Presorted Key: a + -> Parallel Index Scan using t_a_idx on t +(14 rows) -- Incremental sort vs. set operations with varno 0 set enable_hashagg to off; @@ -1580,8 +1582,8 @@ from tenk1 t1 join tenk1 t2 on t1.unique1 = t2.unique2 join tenk1 t3 on t2.unique1 = t3.unique1 order by count(*); - QUERY PLAN ------------------------------------------------------------------------------------------------ + QUERY PLAN +---------------------------------------------------------------------------------------------------- Sort Sort Key: (count(*)) -> Finalize Aggregate @@ -1591,10 +1593,10 @@ order by count(*); -> Parallel Hash Join Hash Cond: (t2.unique1 = t3.unique1) -> Parallel Hash Join - Hash Cond: (t1.unique1 = t2.unique2) - -> Parallel Index Only Scan using tenk1_unique1 on tenk1 t1 + Hash Cond: (t2.unique2 = t1.unique1) + -> Parallel Index Scan using tenk1_unique2 on tenk1 t2 -> Parallel Hash - -> Parallel Index Scan using tenk1_unique2 on tenk1 t2 + -> Parallel Index Only Scan using tenk1_unique1 on tenk1 t1 -> Parallel Hash -> Parallel Index Only Scan using tenk1_unique1 on tenk1 t3 (15 rows) diff --git a/src/test/regress/expected/join_hash.out b/src/test/regress/expected/join_hash.out index 4749f6ed70d..bc7cc76467f 100644 --- a/src/test/regress/expected/join_hash.out +++ b/src/test/regress/expected/join_hash.out @@ -76,8 +76,8 @@ insert into extremely_skewed update pg_class set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192 where relname = 'extremely_skewed'; --- Make a relation with a couple of enormous tuples. -create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +-- Make a relation with several enormous tuples. +create table wide as select generate_series(1, 3) as id, rpad('', 320000, 'x') as t; alter table wide set (parallel_workers = 2); -- The "optimal" case: the hash table fits in memory; we plan for 1 -- batch, we stick to that number, and peak memory usage stays within @@ -922,7 +922,7 @@ set work_mem = '128kB'; set hash_mem_multiplier = 1.0; explain (costs off) select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); QUERY PLAN ---------------------------------------------------------------- Finalize Aggregate @@ -934,10 +934,11 @@ explain (costs off) -> Parallel Seq Scan on wide -> Parallel Hash -> Parallel Seq Scan on wide wide_1 -(9 rows) + Filter: (id < 3) +(10 rows) select length(max(s.t)) -from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); +from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); length -------- 320000 @@ -947,7 +948,7 @@ select final > 1 as multibatch from hash_join_batches( $$ select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); $$); multibatch ------------ diff --git a/src/test/regress/sql/join_hash.sql b/src/test/regress/sql/join_hash.sql index 49d3fd61856..53db1754bb2 100644 --- a/src/test/regress/sql/join_hash.sql +++ b/src/test/regress/sql/join_hash.sql @@ -83,8 +83,8 @@ update pg_class set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192 where relname = 'extremely_skewed'; --- Make a relation with a couple of enormous tuples. -create table wide as select generate_series(1, 2) as id, rpad('', 320000, 'x') as t; +-- Make a relation with several enormous tuples. +create table wide as select generate_series(1, 3) as id, rpad('', 320000, 'x') as t; alter table wide set (parallel_workers = 2); -- The "optimal" case: the hash table fits in memory; we plan for 1 @@ -496,14 +496,14 @@ set work_mem = '128kB'; set hash_mem_multiplier = 1.0; explain (costs off) select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); select length(max(s.t)) -from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); +from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); select final > 1 as multibatch from hash_join_batches( $$ select length(max(s.t)) - from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); + from wide left join (select id, coalesce(t, '') || '' as t from wide where id < 3) s using (id); $$); rollback to settings; -- 2.51.0 ^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Consider low startup cost in add_partial_path @ 2026-03-05 18:17 Robert Haas <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 1 reply; 27+ messages in thread From: Robert Haas @ 2026-03-05 18:17 UTC (permalink / raw) To: Richard Guo <[email protected]>; +Cc: Tomas Vondra <[email protected]>; James Coleman <[email protected]>; pgsql-hackers On Wed, Mar 4, 2026 at 5:04 PM Robert Haas <[email protected]> wrote: > Here's v57. I've instrumented the length of the partial pathlist across all RelOptInfo objects in a run of the core regression test suite. Here's the result: without the patch: 1->9343 2->5859 3->38 with the v57 patch: 1->9306 2->5877 3->51 4->4 5->2 That seems very reasonable to me. It means that, across the test suite, we retain 0.3% more paths than before, and the percentage of partial pathlists of length 1 drops from 61.31% to 61.06%. That is a very modest impact, and it has the upside that in some cases we might get better plans, which is worth a lot. In fact, looking through the regression test changes in the patch, the changes to incremental_sort.out look like improvements: in the first hunk, we now push a PartialAggregate beneath Gather Merge, allowing more work to be done in parallel. In the second hunk, we now put the cheaper scan on the inner side of a Parallel Hash Join instead of the more expensive one. Since the inner side is the build table, the total cost of that side becomes startup cost for hash join, so this change is theoretically better. The total cost of the Parallel Hash Join remains unchanged, while its startup cost drops from 612.32 to 264.32. In this instance, that doesn't improve the startup or total cost for the plan as a whole, but I believe that's just a consequence of the fact that this particular query happens to be putting all of the rows into a single group. In other cases, it might help. The case in join_hash.out is similar: it's testing a join between two tables which, at present, are exactly the same, so even the tiniest perturbation in the costing formulas can upset the plan and cause it not to test the right thing any more. In the patch, I change it so that one side of the join has 2 rows and the other side has 3 rows, and it then puts the 2-row side on the inner side of the parallel hash join for the same reasons discussed in the previous paragraph. This should work just as before but be more stable with these changes. I feel like I've done all the due diligence that I can think of here to be sure that this is correct and doesn't cause a path explosion or plan regressions, so I plan to go ahead and commit this unless there are objections RSN. Thanks, -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Consider low startup cost in add_partial_path @ 2026-03-09 13:08 Robert Haas <[email protected]> parent: Robert Haas <[email protected]> 0 siblings, 0 replies; 27+ messages in thread From: Robert Haas @ 2026-03-09 13:08 UTC (permalink / raw) To: Richard Guo <[email protected]>; +Cc: Tomas Vondra <[email protected]>; James Coleman <[email protected]>; pgsql-hackers On Thu, Mar 5, 2026 at 1:17 PM Robert Haas <[email protected]> wrote: > I feel like I've done all the due diligence that I can think of here > to be sure that this is correct and doesn't cause a path explosion or > plan regressions, so I plan to go ahead and commit this unless there > are objections RSN. Committed. -- Robert Haas EDB: http://www.enterprisedb.com ^ permalink raw reply [nested|flat] 27+ messages in thread
end of thread, other threads:[~2026-03-09 13:08 UTC | newest] Thread overview: 27+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2021-02-01 15:28 [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions. Heikki Linnakangas <[email protected]> 2026-02-19 19:29 Re: Consider low startup cost in add_partial_path Robert Haas <[email protected]> 2026-02-25 06:35 ` Re: Consider low startup cost in add_partial_path Richard Guo <[email protected]> 2026-03-04 22:04 ` Re: Consider low startup cost in add_partial_path Robert Haas <[email protected]> 2026-03-05 18:17 ` Re: Consider low startup cost in add_partial_path Robert Haas <[email protected]> 2026-03-09 13:08 ` Re: Consider low startup cost in add_partial_path Robert Haas <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox