public inbox for [email protected]
help / color / mirror / Atom feedFrom: Heikki Linnakangas <[email protected]>
Subject: [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions.
Date: Mon, 1 Feb 2021 17:28:31 +0200
TODO: When this is pushed, it probably makes more sense to add the
whole regression as one commit after the code changes. But this is perhaps
useful to keep as a separate commit for now.
---
src/backend/utils/mb/mbutils.c | 55 ++
src/include/mb/pg_wchar.h | 6 +
src/test/regress/expected/conversion.out | 575 +++++++++---------
.../regress/input/create_function_1.source | 4 +
.../regress/output/create_function_1.source | 3 +
src/test/regress/regress.c | 121 ++++
src/test/regress/sql/conversion.sql | 17 +-
7 files changed, 482 insertions(+), 299 deletions(-)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 65753860e35..3e106027d75 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -436,6 +436,61 @@ pg_do_encoding_conversion(unsigned char *src, int len,
return result;
}
+/*
+ * Convert src string to another encoding.
+ *
+ * This function has a different API than the other conversion functions.
+ * The caller should've looked up the conversion function using
+ * FindDefaultConversionProc(). Unlike the other functions, the converted
+ * result is not palloc'd. It is written to a caller-supplied buffer instead.
+ *
+ * src_encoding - encoding to convert from
+ * dest_encoding - encoding to convert to
+ * src, srclen - input buffer and its length in bytes
+ * dest, destlen - destination buffer and its size in bytes
+ *
+ * The output is null-terminated.
+ *
+ * If destlen < srclen * MAX_CONVERSION_LENGTH + 1, the converted output
+ * wouldn't necessarily fit in the output buffer, and the function will not
+ * convert the whole input.
+ *
+ * TODO: It would be nice to also return the number of bytes written to the
+ * caller, to avoid a call to strlen().
+ */
+int
+pg_do_encoding_conversion_buf(Oid proc,
+ int src_encoding,
+ int dest_encoding,
+ unsigned char *src, int srclen,
+ unsigned char *dest, int destlen,
+ bool noError)
+{
+ Datum result;
+
+ /*
+ * If the destination buffer is not large enough to hold the
+ * result in the worst case, limit the input size passed to
+ * the conversion function.
+ *
+ * TODO: It would perhaps be more efficient to pass the destination
+ * buffer size to the conversion function, so that if the conversion
+ * expands less than the worst case, it could continue to fill up the
+ * whole buffer.
+ */
+ if ((Size) srclen >= ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH))
+ srclen = ((destlen - 1) / (Size) MAX_CONVERSION_GROWTH);
+
+ result = OidFunctionCall6(proc,
+ Int32GetDatum(src_encoding),
+ Int32GetDatum(dest_encoding),
+ CStringGetDatum(src),
+ CStringGetDatum(dest),
+ Int32GetDatum(srclen),
+ BoolGetDatum(noError));
+ return DatumGetInt32(result);
+}
+
/*
* Convert string to encoding encoding_name. The source
* encoding is the DB encoding.
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 346a41a1f3d..9a22a6461d6 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -616,6 +616,12 @@ extern int pg_bind_textdomain_codeset(const char *domainname);
extern unsigned char *pg_do_encoding_conversion(unsigned char *src, int len,
int src_encoding,
int dest_encoding);
+extern int pg_do_encoding_conversion_buf(Oid proc,
+ int src_encoding,
+ int dest_encoding,
+ unsigned char *src, int srclen,
+ unsigned char *dst, int dstlen,
+ bool noError);
extern char *pg_client_to_server(const char *s, int len);
extern char *pg_server_to_client(const char *s, int len);
diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out
index 38f8cef0f38..571815683e9 100644
--- a/src/test/regress/expected/conversion.out
+++ b/src/test/regress/expected/conversion.out
@@ -53,24 +53,19 @@ $$
declare
validlen int;
begin
- -- Try to perform the conversion. If it fails, catch the error and return
- -- it to the caller.
begin
- select * into result from convert(input, src_encoding, dst_encoding);
- validlen = length(input);
+ select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false);
errorat = NULL;
error := NULL;
exception when others then
- result = NULL;
- errorat = NULL;
+ select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true);
+ errorat = substr(input, validlen + 1);
error := sqlerrm;
end;
return;
end;
$$;
---
--- UTF-8
---
+-- Test verification functions
CREATE TABLE utf8_inputs (inbytes bytea, description text);
insert into utf8_inputs values
('\x666f6f', 'valid, pure ASCII'),
@@ -87,123 +82,123 @@ insert into utf8_inputs values
('\x666f6fe8b1', 'incomplete character at end');
-- Test UTF-8 verification
select description, (test_conv(inbytes, 'utf8', 'utf8')).* from utf8_inputs;
- description | result | errorat | error
-------------------------------------------------------+----------------------+---------+-----------------------------------------------------------
- valid, pure ASCII | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | |
- valid, cyrillic | \xd184d0bed0be | |
- valid, kanji/Chinese | \x666f6fe8b1a1 | |
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | |
- only first half of combined char in EUC_JIS_2004 | \xe382ab | |
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | |
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | |
- invalid byte sequence | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | result | errorat | error
+------------------------------------------------------+----------------------+--------------+-----------------------------------------------------------
+ valid, pure ASCII | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | |
+ valid, cyrillic | \xd184d0bed0be | |
+ valid, kanji/Chinese | \x666f6fe8b1a1 | |
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | |
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | |
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | |
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | |
+ invalid byte sequence | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
-- Test conversions from UTF-8
select description, inbytes, (test_conv(inbytes, 'utf8', 'euc_jis_2004')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------------+---------+-------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | |
- valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | |
- valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | |
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | |
- only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | |
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004"
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004"
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------------+----------------------+-------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \xa9daa9ec | |
+ valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | |
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fbedd | |
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5f7 | |
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | |
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004"
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "EUC_JIS_2004"
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
select description, inbytes, (test_conv(inbytes, 'utf8', 'latin1')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------+---------+-------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
- valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1"
- valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1"
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1"
- only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1"
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1"
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1"
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------+----------------------+-------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
+ valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN1"
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
select description, inbytes, (test_conv(inbytes, 'utf8', 'latin2')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------+---------+-------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
- valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2"
- valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2"
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2"
- only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2"
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2"
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2"
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------+----------------------+-------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
+ valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN2"
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
select description, inbytes, (test_conv(inbytes, 'utf8', 'latin5')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------+---------+-------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
- valid, cyrillic | \xd184d0bed0be | | | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5"
- valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5"
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5"
- only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5"
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5"
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5"
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------+----------------------+-------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \xe4f6 | |
+ valid, cyrillic | \xd184d0bed0be | \x | \xd184d0bed0be | character with byte sequence 0xd1 0x84 in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "LATIN5"
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
select description, inbytes, (test_conv(inbytes, 'utf8', 'koi8r')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------+---------+------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | | | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R"
- valid, cyrillic | \xd184d0bed0be | \xc6cfcf | |
- valid, kanji/Chinese | \x666f6fe8b1a1 | | | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R"
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R"
- only first half of combined char in EUC_JIS_2004 | \xe382ab | | | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R"
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | | | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R"
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | | | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R"
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------+----------------------+------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \x | \xc3a4c3b6 | character with byte sequence 0xc3 0xa4 in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ valid, cyrillic | \xd184d0bed0be | \xc6cfcf | |
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6f | \xe8b1a1 | character with byte sequence 0xe8 0xb1 0xa1 in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \x | \xe382abe3829a | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \x | \xe382ab | character with byte sequence 0xe3 0x82 0xab in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x | \xecbd94eb81bceba6ac | character with byte sequence 0xec 0xbd 0x94 in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f | \xefa8aa | character with byte sequence 0xef 0xa8 0xaa in encoding "UTF8" has no equivalent in encoding "KOI8R"
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
select description, inbytes, (test_conv(inbytes, 'utf8', 'gb18030')).* from utf8_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------------+----------------------+----------------------------+---------+-----------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | |
- valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | |
- valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | |
- valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | |
- only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | |
- valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | |
- valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | |
- invalid byte sequence | \x66e8b1ff6f6f | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
- invalid, NUL byte | \x66006f | | | invalid byte sequence for encoding "UTF8": 0x00
- invalid, NUL byte | \x666f6fe8b100 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
- incomplete character at end | \x666f6fe8b1 | | | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
+ description | inbytes | result | errorat | error
+------------------------------------------------------+----------------------+----------------------------+--------------+-----------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid, extra latin chars | \xc3a4c3b6 | \x81308a3181308b32 | |
+ valid, cyrillic | \xd184d0bed0be | \xa7e6a7e0a7e0 | |
+ valid, kanji/Chinese | \x666f6fe8b1a1 | \x666f6fcff3 | |
+ valid, two chars that combine to one in EUC_JIS_2004 | \xe382abe3829a | \xa5ab8139a732 | |
+ only first half of combined char in EUC_JIS_2004 | \xe382ab | \xa5ab | |
+ valid, Hangul, Korean | \xecbd94eb81bceba6ac | \x8334e5398238c4338330b335 | |
+ valid, needs mapping function to convert to GB18030 | \x666f6fefa8aa | \x666f6f84309c38 | |
+ invalid byte sequence | \x66e8b1ff6f6f | \x66 | \xe8b1ff6f6f | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0xff
+ invalid, NUL byte | \x66006f | \x66 | \x006f | invalid byte sequence for encoding "UTF8": 0x00
+ invalid, NUL byte | \x666f6fe8b100 | \x666f6f | \xe8b100 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1 0x00
+ incomplete character at end | \x666f6fe8b1 | \x666f6f | \xe8b1 | invalid byte sequence for encoding "UTF8": 0xe8 0xb1
(12 rows)
--
@@ -221,30 +216,30 @@ insert into euc_jis_2004_inputs values
('\xbe04', 'invalid byte sequence');
-- Test EUC_JIS_2004 verification
select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'euc_jis_2004')).* from euc_jis_2004_inputs;
- description | inbytes | result | errorat | error
----------------------------------------+----------------+--------------+---------+--------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fbedd | \x666f6fbedd | |
- valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | |
- incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe
- invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
- invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00
- invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
- invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04
+ description | inbytes | result | errorat | error
+---------------------------------------+----------------+--------------+----------+--------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fbedd | \x666f6fbedd | |
+ valid, translates to two UTF-8 chars | \xa5f7 | \xa5f7 | |
+ incomplete char | \xbeddbe | \xbedd | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe
+ invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
+ invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00
+ invalid, NUL byte | \x666f6fbedd00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
+ invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04
(8 rows)
-- Test conversions from EUC_JIS_2004
select description, inbytes, (test_conv(inbytes, 'euc_jis_2004', 'utf8')).* from euc_jis_2004_inputs;
- description | inbytes | result | errorat | error
----------------------------------------+----------------+----------------+---------+--------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fbedd | \x666f6fe8b1a1 | |
- valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | |
- incomplete char | \xbeddbe | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe
- invalid, NUL byte | \x666f6f00bedd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
- invalid, NUL byte | \x666f6fbe00dd | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00
- invalid, NUL byte | \x666f6fbedd00 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
- invalid byte sequence | \xbe04 | | | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04
+ description | inbytes | result | errorat | error
+---------------------------------------+----------------+----------------+----------+--------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fbedd | \x666f6fe8b1a1 | |
+ valid, translates to two UTF-8 chars | \xa5f7 | \xe382abe3829a | |
+ incomplete char | \xbeddbe | \xe8b1a1 | \xbe | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe
+ invalid, NUL byte | \x666f6f00bedd | \x666f6f | \x00bedd | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
+ invalid, NUL byte | \x666f6fbe00dd | \x666f6f | \xbe00dd | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x00
+ invalid, NUL byte | \x666f6fbedd00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "EUC_JIS_2004": 0x00
+ invalid byte sequence | \xbe04 | \x | \xbe04 | invalid byte sequence for encoding "EUC_JIS_2004": 0xbe 0x04
(8 rows)
--
@@ -263,46 +258,46 @@ insert into shiftjis2004_inputs values
('\x666f6f8fdb00', 'invalid, NUL byte');
-- Test SHIFT-JIS-2004 verification
select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'shiftjis2004')).* from shiftjis2004_inputs;
- description | inbytes | result | errorat | error
----------------------------------------+----------------+--------------+---------+----------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6f8fdb | \x666f6f8fdb | |
- valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | |
- valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | |
- incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
- incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
- invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
- invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
- invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ description | inbytes | result | errorat | error
+---------------------------------------+----------------+--------------+----------+----------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6f8fdb | \x666f6f8fdb | |
+ valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6f81c0 | |
+ valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6f82f5 | |
+ incomplete char | \x666f6f8fdb8f | \x666f6f8fdb | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
+ incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
+ invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
+ invalid, NUL byte | \x666f6f8fdb00 | \x666f6f8fdb | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
(9 rows)
-- Test conversions from SHIFT-JIS-2004
select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'utf8')).* from shiftjis2004_inputs;
- description | inbytes | result | errorat | error
----------------------------------------+----------------+----------------------+---------+----------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6f8fdb | \x666f6fe8b1a1 | |
- valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | |
- valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | |
- incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
- incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
- invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
- invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
- invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ description | inbytes | result | errorat | error
+---------------------------------------+----------------+----------------------+----------+----------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6f8fdb | \x666f6fe8b1a1 | |
+ valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fe28a84 | |
+ valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fe3818be3829a | |
+ incomplete char | \x666f6f8fdb8f | \x666f6fe8b1a1 | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
+ incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
+ invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
+ invalid, NUL byte | \x666f6f8fdb00 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
(9 rows)
select description, inbytes, (test_conv(inbytes, 'shiftjis2004', 'euc_jis_2004')).* from shiftjis2004_inputs;
- description | inbytes | result | errorat | error
----------------------------------------+----------------+--------------+---------+----------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6f8fdb | \x666f6fbedd | |
- valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | |
- valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | |
- incomplete char | \x666f6f8fdb8f | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
- incomplete char, followed by newline | \x666f6f820a | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
- invalid, NUL byte | \x666f6f008fdb | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
- invalid, NUL byte | \x666f6f8f00db | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
- invalid, NUL byte | \x666f6f8fdb00 | | | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ description | inbytes | result | errorat | error
+---------------------------------------+----------------+--------------+----------+----------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6f8fdb | \x666f6fbedd | |
+ valid, no translation to UTF-8 | \x666f6f81c0 | \x666f6fa2c2 | |
+ valid, translates to two UTF-8 chars | \x666f6f82f5 | \x666f6fa4f7 | |
+ incomplete char | \x666f6f8fdb8f | \x666f6fbedd | \x8f | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f
+ incomplete char, followed by newline | \x666f6f820a | \x666f6f | \x820a | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x82 0x0a
+ invalid, NUL byte | \x666f6f008fdb | \x666f6f | \x008fdb | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
+ invalid, NUL byte | \x666f6f8f00db | \x666f6f | \x8f00db | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x8f 0x00
+ invalid, NUL byte | \x666f6f8fdb00 | \x666f6fbedd | \x00 | invalid byte sequence for encoding "SHIFT_JIS_2004": 0x00
(9 rows)
--
@@ -320,30 +315,30 @@ insert into gb18030_inputs values
('\x666f6f84309c0038', 'invalid, NUL byte');
-- Test GB18030 verification
select description, inbytes, (test_conv(inbytes, 'gb18030', 'gb18030')).* from gb18030_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------+--------------------+------------------+---------+-------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fcff3 | \x666f6fcff3 | |
- valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | |
- valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | |
- incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c
- incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a
- invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00
- invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00
+ description | inbytes | result | errorat | error
+------------------------------------------------+--------------------+------------------+--------------+-------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fcff3 | \x666f6fcff3 | |
+ valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f8431a530 | |
+ valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6f84309c38 | |
+ incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c
+ incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a
+ invalid, NUL byte | \x666f6f84309c3800 | \x666f6f84309c38 | \x00 | invalid byte sequence for encoding "GB18030": 0x00
+ invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00
(8 rows)
-- Test conversions from GB18030
select description, inbytes, (test_conv(inbytes, 'gb18030', 'utf8')).* from gb18030_inputs;
- description | inbytes | result | errorat | error
-------------------------------------------------+--------------------+----------------+---------+-------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fcff3 | \x666f6fe8b1a1 | |
- valid, no translation to UTF-8 | \x666f6f8431a530 | | | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8"
- valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | |
- incomplete char | \x666f6f84309c | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c
- incomplete char, followed by newline | \x666f6f84309c0a | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a
- invalid, NUL byte | \x666f6f84309c3800 | | | invalid byte sequence for encoding "GB18030": 0x00
- invalid, NUL byte | \x666f6f84309c0038 | | | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00
+ description | inbytes | result | errorat | error
+------------------------------------------------+--------------------+----------------+--------------+-------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fcff3 | \x666f6fe8b1a1 | |
+ valid, no translation to UTF-8 | \x666f6f8431a530 | \x666f6f | \x8431a530 | character with byte sequence 0x84 0x31 0xa5 0x30 in encoding "GB18030" has no equivalent in encoding "UTF8"
+ valid, translates to UTF-8 by mapping function | \x666f6f84309c38 | \x666f6fefa8aa | |
+ incomplete char | \x666f6f84309c | \x666f6f | \x84309c | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c
+ incomplete char, followed by newline | \x666f6f84309c0a | \x666f6f | \x84309c0a | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x0a
+ invalid, NUL byte | \x666f6f84309c3800 | \x666f6fefa8aa | \x00 | invalid byte sequence for encoding "GB18030": 0x00
+ invalid, NUL byte | \x666f6f84309c0038 | \x666f6f | \x84309c0038 | invalid byte sequence for encoding "GB18030": 0x84 0x30 0x9c 0x00
(8 rows)
--
@@ -358,44 +353,44 @@ insert into iso8859_5_inputs values
('\xe4dede00', 'invalid, NUL byte');
-- Test ISO-88591 verification
select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'iso8859-5')).* from iso8859_5_inputs;
- description | inbytes | result | errorat | error
--------------------+------------+----------+---------+-------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \xe4dede | \xe4dede | |
- invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ description | inbytes | result | errorat | error
+-------------------+------------+----------+----------+-------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \xe4dede | \xe4dede | |
+ invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe400dede | \xe4 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe4dede00 | \xe4dede | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
(5 rows)
-- Test conversions from ISO-88591
select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'utf8')).* from iso8859_5_inputs;
- description | inbytes | result | errorat | error
--------------------+------------+----------------+---------+-------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \xe4dede | \xd184d0bed0be | |
- invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ description | inbytes | result | errorat | error
+-------------------+------------+----------------+----------+-------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \xe4dede | \xd184d0bed0be | |
+ invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe400dede | \xd184 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe4dede00 | \xd184d0bed0be | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
(5 rows)
select description, inbytes, (test_conv(inbytes, 'iso8859-5', 'koi8r')).* from iso8859_5_inputs;
- description | inbytes | result | errorat | error
--------------------+------------+----------+---------+-------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \xe4dede | \xc6cfcf | |
- invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ description | inbytes | result | errorat | error
+-------------------+------------+----------+----------+-------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \xe4dede | \xc6cfcf | |
+ invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe400dede | \xc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe4dede00 | \xc6cfcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
(5 rows)
select description, inbytes, (test_conv(inbytes, 'iso8859_5', 'mule_internal')).* from iso8859_5_inputs;
- description | inbytes | result | errorat | error
--------------------+------------+----------------+---------+-------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \xe4dede | \x8bc68bcf8bcf | |
- invalid, NUL byte | \x00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe400dede | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
- invalid, NUL byte | \xe4dede00 | | | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ description | inbytes | result | errorat | error
+-------------------+------------+----------------+----------+-------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \xe4dede | \x8bc68bcf8bcf | |
+ invalid, NUL byte | \x00 | \x | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe400dede | \x8bc6 | \x00dede | invalid byte sequence for encoding "ISO_8859_5": 0x00
+ invalid, NUL byte | \xe4dede00 | \x8bc68bcf8bcf | \x00 | invalid byte sequence for encoding "ISO_8859_5": 0x00
(5 rows)
--
@@ -410,37 +405,39 @@ insert into big5_inputs values
('\x666f6fb64800', 'invalid, NUL byte');
-- Test Big5 verification
select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_inputs;
- description | inbytes | result | errorat | error
---------------------------------+----------------+--------------+---------+------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fb648 | \x666f6fb648 | |
- valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | |
- invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00
- invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00
+ description | inbytes | result | errorat | error
+--------------------------------+----------------+--------------+----------+------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fb648 | \x666f6fb648 | |
+ valid, no translation to UTF-8 | \x666f6fa27f | \x666f6fa27f | |
+ invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00
+ invalid, NUL byte | \x666f6fb64800 | \x666f6fb648 | \x00 | invalid byte sequence for encoding "BIG5": 0x00
(5 rows)
-- Test conversions from Big5
select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs;
- description | inbytes | result | errorat | error
---------------------------------+----------------+----------------+---------+------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fb648 | \x666f6fe8b1a1 | |
- valid, no translation to UTF-8 | \x666f6fa27f | | | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8"
- invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00
- invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00
+ description | inbytes | result | errorat | error
+--------------------------------+----------------+----------------+----------+------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fb648 | \x666f6fe8b1a1 | |
+ valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f | \xa27f | character with byte sequence 0xa2 0x7f in encoding "BIG5" has no equivalent in encoding "UTF8"
+ invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00
+ invalid, NUL byte | \x666f6fb64800 | \x666f6fe8b1a1 | \x00 | invalid byte sequence for encoding "BIG5": 0x00
(5 rows)
select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs;
- description | inbytes | result | errorat | error
---------------------------------+----------------+----------------+---------+------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid | \x666f6fb648 | \x666f6f95e2af | |
- valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | |
- invalid, NUL byte | \x666f6fb60048 | | | invalid byte sequence for encoding "BIG5": 0xb6 0x00
- invalid, NUL byte | \x666f6fb64800 | | | invalid byte sequence for encoding "BIG5": 0x00
+ description | inbytes | result | errorat | error
+--------------------------------+----------------+----------------+----------+------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid | \x666f6fb648 | \x666f6f95e2af | |
+ valid, no translation to UTF-8 | \x666f6fa27f | \x666f6f95a3c1 | |
+ invalid, NUL byte | \x666f6fb60048 | \x666f6f | \xb60048 | invalid byte sequence for encoding "BIG5": 0xb6 0x00
+ invalid, NUL byte | \x666f6fb64800 | \x666f6f95e2af | \x00 | invalid byte sequence for encoding "BIG5": 0x00
(5 rows)
+--
-- MULE_INTERNAL
+--
CREATE TABLE mic_inputs (inbytes bytea, description text);
insert into mic_inputs values
('\x666f6f', 'valid, pure ASCII'),
@@ -455,78 +452,78 @@ insert into mic_inputs values
('\x8b00c68bcf8bcf', 'invalid, NUL byte');
-- Test MULE_INTERNAL verification
select description, inbytes, (test_conv(inbytes, 'mule_internal', 'koi8r')).* from mic_inputs;
- description | inbytes | result | errorat | error
----------------------------+------------------+----------+---------+---------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | |
- invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
- valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
- invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
- valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
- invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
- invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
- invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
- invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
+ description | inbytes | result | errorat | error
+---------------------------+------------------+----------+------------------+---------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid (in KOI8R) | \x8bc68bcf8bcf | \xc6cfcf | |
+ invalid,incomplete char | \x8bc68bcf8b | \xc6cf | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
+ valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
+ invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
+ valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
+ invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
+ invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
+ invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
+ invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "KOI8R"
(10 rows)
-- Test conversions from MULE_INTERNAL
select description, inbytes, (test_conv(inbytes, 'mule_internal', 'iso8859-5')).* from mic_inputs;
- description | inbytes | result | errorat | error
----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | |
- invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
- valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
- invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
- valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
- invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
- invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
- invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
- invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
+ description | inbytes | result | errorat | error
+---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid (in KOI8R) | \x8bc68bcf8bcf | \xe4dede | |
+ invalid,incomplete char | \x8bc68bcf8b | \xe4de | \x8b | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
+ valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
+ invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
+ valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
+ invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
+ invalid, NUL byte | \x9200bedd | \x | \x9200bedd | character with byte sequence 0x92 0x00 0xbe in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
+ invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
+ invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | character with byte sequence 0x8b 0x00 in encoding "MULE_INTERNAL" has no equivalent in encoding "ISO_8859_5"
(10 rows)
select description, inbytes, (test_conv(inbytes, 'mule_internal', 'sjis')).* from mic_inputs;
- description | inbytes | result | errorat | error
----------------------------+------------------+----------+---------+--------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS"
- invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
- valid (in SHIFT_JIS) | \x92bedd | \x8fdb | |
- invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
- valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS"
- invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
- invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
- invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
- invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
+ description | inbytes | result | errorat | error
+---------------------------+------------------+----------+------------------+--------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS"
+ invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS"
+ valid (in SHIFT_JIS) | \x92bedd | \x8fdb | |
+ invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
+ valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "SJIS"
+ invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
+ invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
+ invalid, NUL byte | \x92bedd00 | \x8fdb | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
+ invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
(10 rows)
select description, inbytes, (test_conv(inbytes, 'mule_internal', 'big5')).* from mic_inputs;
- description | inbytes | result | errorat | error
----------------------------+------------------+--------------+---------+--------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
- invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
- valid (in SHIFT_JIS) | \x92bedd | | | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
- invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
- valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | |
- invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
- invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
- invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
- invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
+ description | inbytes | result | errorat | error
+---------------------------+------------------+--------------+------------------+--------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
+ invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
+ valid (in SHIFT_JIS) | \x92bedd | \x | \x92bedd | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
+ invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
+ valid (in Big5) | \x666f6f95a3c1 | \x666f6fa2a1 | |
+ invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
+ invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
+ invalid, NUL byte | \x92bedd00 | \x | \x92bedd00 | character with byte sequence 0x92 0xbe 0xdd in encoding "MULE_INTERNAL" has no equivalent in encoding "BIG5"
+ invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
(10 rows)
select description, inbytes, (test_conv(inbytes, 'mule_internal', 'euc_jp')).* from mic_inputs;
- description | inbytes | result | errorat | error
----------------------------+------------------+----------+---------+----------------------------------------------------------------------------------------------------------------
- valid, pure ASCII | \x666f6f | \x666f6f | |
- valid (in KOI8R) | \x8bc68bcf8bcf | | | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP"
- invalid,incomplete char | \x8bc68bcf8b | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b
- valid (in SHIFT_JIS) | \x92bedd | \xbedd | |
- invalid, incomplete char) | \x92be | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
- valid (in Big5) | \x666f6f95a3c1 | | | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP"
- invalid, incomplete char | \x666f6f95a3 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
- invalid, NUL byte | \x9200bedd | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
- invalid, NUL byte | \x92bedd00 | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
- invalid, NUL byte | \x8b00c68bcf8bcf | | | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
+ description | inbytes | result | errorat | error
+---------------------------+------------------+----------+------------------+----------------------------------------------------------------------------------------------------------------
+ valid, pure ASCII | \x666f6f | \x666f6f | |
+ valid (in KOI8R) | \x8bc68bcf8bcf | \x | \x8bc68bcf8bcf | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP"
+ invalid,incomplete char | \x8bc68bcf8b | \x | \x8bc68bcf8b | character with byte sequence 0x8b 0xc6 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP"
+ valid (in SHIFT_JIS) | \x92bedd | \xbedd | |
+ invalid, incomplete char) | \x92be | \x | \x92be | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0xbe
+ valid (in Big5) | \x666f6f95a3c1 | \x666f6f | \x95a3c1 | character with byte sequence 0x95 0xa3 0xc1 in encoding "MULE_INTERNAL" has no equivalent in encoding "EUC_JP"
+ invalid, incomplete char | \x666f6f95a3 | \x666f6f | \x95a3 | invalid byte sequence for encoding "MULE_INTERNAL": 0x95 0xa3
+ invalid, NUL byte | \x9200bedd | \x | \x9200bedd | invalid byte sequence for encoding "MULE_INTERNAL": 0x92 0x00 0xbe
+ invalid, NUL byte | \x92bedd00 | \xbedd | \x00 | invalid byte sequence for encoding "MULE_INTERNAL": 0x00
+ invalid, NUL byte | \x8b00c68bcf8bcf | \x | \x8b00c68bcf8bcf | invalid byte sequence for encoding "MULE_INTERNAL": 0x8b 0x00
(10 rows)
diff --git a/src/test/regress/input/create_function_1.source b/src/test/regress/input/create_function_1.source
index 412e339fcf2..6ba37fe63b6 100644
--- a/src/test/regress/input/create_function_1.source
+++ b/src/test/regress/input/create_function_1.source
@@ -78,6 +78,10 @@ CREATE FUNCTION test_opclass_options_func(internal)
AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func'
LANGUAGE C;
+CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea)
+ AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion'
+ LANGUAGE C;
+
-- Things that shouldn't work:
CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL
diff --git a/src/test/regress/output/create_function_1.source b/src/test/regress/output/create_function_1.source
index 4d78fa12289..cb38a039bf4 100644
--- a/src/test/regress/output/create_function_1.source
+++ b/src/test/regress/output/create_function_1.source
@@ -68,6 +68,9 @@ CREATE FUNCTION test_opclass_options_func(internal)
RETURNS void
AS '@libdir@/regress@DLSUFFIX@', 'test_opclass_options_func'
LANGUAGE C;
+CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea)
+ AS '@libdir@/regress@DLSUFFIX@', 'test_enc_conversion'
+ LANGUAGE C;
-- Things that shouldn't work:
CREATE FUNCTION test1 (int) RETURNS int LANGUAGE SQL
AS 'SELECT ''not an integer'';';
diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c
index 32ab9ed6b53..6dc43ff7fea 100644
--- a/src/test/regress/regress.c
+++ b/src/test/regress/regress.c
@@ -23,12 +23,15 @@
#include "access/htup_details.h"
#include "access/transam.h"
#include "access/xact.h"
+#include "catalog/namespace.h"
#include "catalog/pg_operator.h"
#include "catalog/pg_type.h"
#include "commands/sequence.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "executor/spi.h"
+#include "funcapi.h"
+#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "nodes/supportnodes.h"
#include "optimizer/optimizer.h"
@@ -1060,3 +1063,121 @@ test_opclass_options_func(PG_FUNCTION_ARGS)
{
PG_RETURN_NULL();
}
+
+
+PG_FUNCTION_INFO_V1(test_enc_conversion);
+Datum
+test_enc_conversion(PG_FUNCTION_ARGS)
+{
+ bytea *string = PG_GETARG_BYTEA_PP(0);
+ char *src_encoding_name = NameStr(*PG_GETARG_NAME(1));
+ int src_encoding = pg_char_to_encoding(src_encoding_name);
+ char *dest_encoding_name = NameStr(*PG_GETARG_NAME(2));
+ int dest_encoding = pg_char_to_encoding(dest_encoding_name);
+ bool noError = PG_GETARG_BOOL(3);
+ TupleDesc tupdesc;
+ char *src;
+ char *dst;
+ bytea *retval;
+ Size srclen;
+ Size dstsize;
+ Oid proc;
+ int convertedbytes;
+ int dstlen;
+ Datum values[2];
+ bool nulls[2];
+ HeapTuple tuple;
+
+ if (src_encoding < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("invalid source encoding name \"%s\"",
+ src_encoding_name)));
+ if (dest_encoding < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("invalid destination encoding name \"%s\"",
+ dest_encoding_name)));
+
+ /* Build a tuple descriptor for our result type */
+ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+ elog(ERROR, "return type must be a row type");
+ tupdesc = BlessTupleDesc(tupdesc);
+
+ srclen = VARSIZE_ANY_EXHDR(string);
+ src = VARDATA_ANY(string);
+
+ if (src_encoding == dest_encoding)
+ {
+ /* just check that the source string is valid */
+ int oklen;
+
+ oklen = pg_encoding_verifymbstr(src_encoding, src, srclen);
+
+ if (oklen == srclen)
+ {
+ convertedbytes = oklen;
+ retval = string;
+ }
+ else if (!noError)
+ {
+ report_invalid_encoding(src_encoding, src + oklen, srclen - oklen);
+ }
+ else
+ {
+ /*
+ * build bytea data type structure.
+ */
+ Assert(oklen < srclen);
+ convertedbytes = oklen;
+ retval = (bytea *) palloc(oklen + VARHDRSZ);
+ SET_VARSIZE(retval, oklen + VARHDRSZ);
+ memcpy(VARDATA(retval), src, oklen);
+ }
+ }
+ else
+ {
+ proc = FindDefaultConversionProc(src_encoding, dest_encoding);
+ if (!OidIsValid(proc))
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_FUNCTION),
+ errmsg("default conversion function for encoding \"%s\" to \"%s\" does not exist",
+ pg_encoding_to_char(src_encoding),
+ pg_encoding_to_char(dest_encoding))));
+
+ if (srclen >= (MaxAllocSize / (Size) MAX_CONVERSION_GROWTH))
+ ereport(ERROR,
+ (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+ errmsg("out of memory"),
+ errdetail("String of %d bytes is too long for encoding conversion.",
+ (int) srclen)));
+
+ dstsize = (Size) srclen * MAX_CONVERSION_GROWTH + 1;
+ dst = MemoryContextAlloc(CurrentMemoryContext, dstsize);
+
+ /* perform conversion */
+ convertedbytes = pg_do_encoding_conversion_buf(proc,
+ src_encoding,
+ dest_encoding,
+ (unsigned char *) src, srclen,
+ (unsigned char *) dst, dstsize,
+ noError);
+ dstlen = strlen(dst);
+
+ /*
+ * build bytea data type structure.
+ */
+ retval = (bytea *) palloc(dstlen + VARHDRSZ);
+ SET_VARSIZE(retval, dstlen + VARHDRSZ);
+ memcpy(VARDATA(retval), dst, dstlen);
+
+ pfree(dst);
+ }
+
+ MemSet(nulls, 0, sizeof(nulls));
+ values[0] = Int32GetDatum(convertedbytes);
+ values[1] = PointerGetDatum(retval);
+ tuple = heap_form_tuple(tupdesc, values, nulls);
+
+ PG_RETURN_DATUM(HeapTupleGetDatum(tuple));
+}
diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql
index 644531d3333..41e2686d46a 100644
--- a/src/test/regress/sql/conversion.sql
+++ b/src/test/regress/sql/conversion.sql
@@ -55,25 +55,21 @@ $$
declare
validlen int;
begin
- -- Try to perform the conversion. If it fails, catch the error and return
- -- it to the caller.
begin
- select * into result from convert(input, src_encoding, dst_encoding);
- validlen = length(input);
+ select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, false);
errorat = NULL;
error := NULL;
exception when others then
- result = NULL;
- errorat = NULL;
+ select * into validlen, result from test_enc_conversion(input, src_encoding, dst_encoding, true);
+ errorat = substr(input, validlen + 1);
error := sqlerrm;
end;
return;
end;
$$;
---
--- UTF-8
---
+-- Test verification functions
+
CREATE TABLE utf8_inputs (inbytes bytea, description text);
insert into utf8_inputs values
('\x666f6f', 'valid, pure ASCII'),
@@ -194,8 +190,9 @@ select description, inbytes, (test_conv(inbytes, 'big5', 'big5')).* from big5_in
select description, inbytes, (test_conv(inbytes, 'big5', 'utf8')).* from big5_inputs;
select description, inbytes, (test_conv(inbytes, 'big5', 'mule_internal')).* from big5_inputs;
-
+--
-- MULE_INTERNAL
+--
CREATE TABLE mic_inputs (inbytes bytea, description text);
insert into mic_inputs values
('\x666f6f', 'valid, pure ASCII'),
--
2.29.2
--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
name="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="v3-0004-Fix-bugs-in-the-commit-to-change-conversion-funct.pa";
filename*1="tch"
view thread (24+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected]
Subject: Re: [PATCH v3 3/5] Add tests for the new noError variants of built-in conversions.
In-Reply-To: <no-message-id-1873501@localhost>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox