pg.ddx.io pgsql-hackers@postgresql.org mailing list archive
help / color / mirror / Atom feedSmall patch to improve safety of utf8_to_unicode().
27+ messages / 4 participants
[nested] [flat]
Commits from this thread:
2026-07-17 2cf212db52 Generate unicode_limits.h.
2026-07-07 07211f64ac unicode_case.c: change API to signal UTF8 decoding error.
2026-07-07 28d498e280 3686936898 66ec24276b pg_unicode_fast: fix final sigma logic.
2026-07-07 21ffc271d4 05da336dd7 9021c8f3ca 5e78ebca57 unicode_case.c: defend against truncated UTF8.
2025-12-15 95a19fefdc Remove incorrect declarations in pg_wchar.h.
* Small patch to improve safety of utf8_to_unicode().
@ 2025-12-12 23:24 Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2025-12-12 23:24 UTC (permalink / raw)
To: pgsql-hackers
Attached.
Attachments:
v1-0001-Make-utf8_to_unicode-safer.patch (text/x-patch, 9.9K) download
From 4777d1af75bc9e570e7db954b6642916fa1420bc Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Dec 2025 19:37:32 -0800
Subject: [PATCH v1] Make utf8_to_unicode() safer.
---
contrib/fuzzystrmatch/daitch_mokotoff.c | 3 ++-
src/backend/utils/adt/pg_locale_builtin.c | 5 +++--
src/backend/utils/adt/varlena.c | 26 +++++++++++------------
src/common/saslprep.c | 4 +++-
src/common/unicode/case_test.c | 5 +++--
src/common/unicode_case.c | 12 +++++++----
src/common/wchar.c | 3 ++-
src/include/mb/pg_wchar.h | 14 +++++-------
8 files changed, 38 insertions(+), 34 deletions(-)
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
index 07f895ae2bf..47bd2814460 100644
--- a/contrib/fuzzystrmatch/daitch_mokotoff.c
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -401,7 +401,8 @@ read_char(const unsigned char *str, int *ix)
/* Decode UTF-8 character to ISO 10646 code point. */
str += *ix;
- c = utf8_to_unicode(str);
+ /* Assume byte sequence has not been broken. */
+ c = utf8_to_unicode(str, MAX_MULTIBYTE_CHAR_LEN);
/* Advance *ix, but (for safety) not if we've reached end of string. */
if (c)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 0d4c754a267..5c1358e5347 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -59,12 +59,13 @@ static size_t
initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
+ unsigned char *str = (unsigned char *) wbstate->str;
while (wbstate->offset < wbstate->len &&
wbstate->str[wbstate->offset] != '\0')
{
- char32_t u = utf8_to_unicode((unsigned char *) wbstate->str +
- wbstate->offset);
+ char32_t u = utf8_to_unicode(str + wbstate->offset,
+ wbstate->len - wbstate->offset);
bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index baa5b44ea8d..d684370900d 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5423,19 +5423,17 @@ Datum
unicode_assigned(PG_FUNCTION_ARGS)
{
text *input = PG_GETARG_TEXT_PP(0);
- unsigned char *p;
- int size;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
if (GetDatabaseEncoding() != PG_UTF8)
ereport(ERROR,
(errmsg("Unicode categorization can only be performed if server encoding is UTF8")));
/* convert to char32_t */
- size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
- p = (unsigned char *) VARDATA_ANY(input);
- for (int i = 0; i < size; i++)
+ while (p < p_end)
{
- char32_t uchar = utf8_to_unicode(p);
+ char32_t uchar = utf8_to_unicode(p, p_end - p);
int category = unicode_category(uchar);
if (category == PG_U_UNASSIGNED)
@@ -5456,7 +5454,8 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
int size;
char32_t *input_chars;
char32_t *output_chars;
- unsigned char *p;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
text *result;
int i;
@@ -5465,14 +5464,13 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
/* convert to char32_t */
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
input_chars = palloc((size + 1) * sizeof(char32_t));
- p = (unsigned char *) VARDATA_ANY(input);
for (i = 0; i < size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert(p == p_end);
/* action */
output_chars = unicode_normalize(form, input_chars);
@@ -5522,7 +5520,8 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
int size;
char32_t *input_chars;
char32_t *output_chars;
- unsigned char *p;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
int i;
UnicodeNormalizationQC quickcheck;
int output_size;
@@ -5533,14 +5532,13 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
/* convert to char32_t */
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
input_chars = palloc((size + 1) * sizeof(char32_t));
- p = (unsigned char *) VARDATA_ANY(input);
for (i = 0; i < size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert(p == p_end);
/* quick check (see UAX #15) */
quickcheck = unicode_is_normalized_quickcheck(form, input_chars);
diff --git a/src/common/saslprep.c b/src/common/saslprep.c
index 101e8d65a4d..083702654b0 100644
--- a/src/common/saslprep.c
+++ b/src/common/saslprep.c
@@ -1055,6 +1055,7 @@ pg_saslprep(const char *input, char **output)
int i;
bool contains_RandALCat;
unsigned char *p;
+ unsigned char *p_end;
char32_t *wp;
/* Ensure we return *output as NULL on failure */
@@ -1088,9 +1089,10 @@ pg_saslprep(const char *input, char **output)
goto oom;
p = (unsigned char *) input;
+ p_end = p + strlen(input);
for (i = 0; i < input_size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 00d4f85e5a5..e63ce2fbeb9 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -51,12 +51,13 @@ static size_t
initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
+ unsigned char *str = (unsigned char *) wbstate->str;
while (wbstate->offset < wbstate->len &&
wbstate->str[wbstate->offset] != '\0')
{
- char32_t u = utf8_to_unicode((unsigned char *) wbstate->str +
- wbstate->offset);
+ char32_t u = utf8_to_unicode(str + wbstate->offset,
+ wbstate->len - wbstate->offset);
bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index e5e494db43c..a760094104c 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -223,15 +223,19 @@ convert_case(char *dst, size_t dstsize, const char *src, ssize_t srclen,
Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
(str_casekind != CaseTitle && !wbnext && !wbstate));
+ if (srclen < 0)
+ srclen = strlen(src);
+
if (str_casekind == CaseTitle)
{
boundary = wbnext(wbstate);
Assert(boundary == 0); /* start of text is always a boundary */
}
- while ((srclen < 0 || srcoff < srclen) && src[srcoff] != '\0')
+ while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((unsigned char *) src + srcoff);
+ char32_t u1 = utf8_to_unicode((unsigned char *) src + srcoff,
+ srclen - srcoff);
int u1len = unicode_utf8len(u1);
char32_t simple = 0;
const char32_t *special = NULL;
@@ -320,7 +324,7 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ char32_t curr = utf8_to_unicode(str + i, len - i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -344,7 +348,7 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ char32_t curr = utf8_to_unicode(str + i, len - i);
if (pg_u_prop_case_ignorable(curr))
continue;
diff --git a/src/common/wchar.c b/src/common/wchar.c
index a4bc29921de..c113cadf815 100644
--- a/src/common/wchar.c
+++ b/src/common/wchar.c
@@ -661,7 +661,8 @@ ucs_wcwidth(pg_wchar ucs)
static int
pg_utf_dsplen(const unsigned char *s)
{
- return ucs_wcwidth(utf8_to_unicode(s));
+ /* trust that input is not a truncated byte sequence */
+ return ucs_wcwidth(utf8_to_unicode(s, MAX_MULTIBYTE_CHAR_LEN));
}
/*
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 4d84bdc81e4..6dc0fff332f 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -558,22 +558,20 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
- *
- * No error checks here, c must point to a long-enough string.
*/
static inline char32_t
-utf8_to_unicode(const unsigned char *c)
+utf8_to_unicode(const unsigned char *c, size_t len)
{
- if ((*c & 0x80) == 0)
+ if ((*c & 0x80) == 0 && len >= 1)
return (char32_t) c[0];
- else if ((*c & 0xe0) == 0xc0)
+ else if ((*c & 0xe0) == 0xc0 && len >= 2)
return (char32_t) (((c[0] & 0x1f) << 6) |
(c[1] & 0x3f));
- else if ((*c & 0xf0) == 0xe0)
+ else if ((*c & 0xf0) == 0xe0 && len >= 3)
return (char32_t) (((c[0] & 0x0f) << 12) |
((c[1] & 0x3f) << 6) |
(c[2] & 0x3f));
- else if ((*c & 0xf8) == 0xf0)
+ else if ((*c & 0xf8) == 0xf0 && len >= 4)
return (char32_t) (((c[0] & 0x07) << 18) |
((c[1] & 0x3f) << 12) |
((c[2] & 0x3f) << 6) |
@@ -676,8 +674,6 @@ extern int pg_valid_server_encoding(const char *name);
extern bool is_encoding_supported_by_icu(int encoding);
extern const char *get_encoding_name_for_icu(int encoding);
-extern unsigned char *unicode_to_utf8(char32_t c, unsigned char *utf8string);
-extern char32_t utf8_to_unicode(const unsigned char *c);
extern bool pg_utf8_islegal(const unsigned char *source, int length);
extern int pg_utf_mblen(const unsigned char *s);
extern int pg_mule_mblen(const unsigned char *s);
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2025-12-13 23:22 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Chao Li @ 2025-12-13 23:22 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Dec 13, 2025, at 07:24, Jeff Davis <pgsql@j-davis.com> wrote:
>
> Attached.
>
>
> <v1-0001-Make-utf8_to_unicode-safer.patch>
This patch adds a length check to utf8_to_unicode(), I think which is where “safety” comes from. Can you please add an a little bit more to the commit message instead of only saying “improve safety”. It also deleted two redundant function declarations from pg_wchar.h, which may also worth a quick note in the commit message.
The code changes all look good to me. Only nitpicks are:
1
```
inline patch 1: contrib/fuzzystrmatch/daitch_mokotoff.c (+2/-1 lines) download inline-1.patch
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
index 07f895ae2bf..47bd2814460 100644
--- a/contrib/fuzzystrmatch/daitch_mokotoff.c
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -401,7 +401,8 @@ read_char(const unsigned char *str, int *ix)
/* Decode UTF-8 character to ISO 10646 code point. */
str += *ix;
- c = utf8_to_unicode(str);
+ /* Assume byte sequence has not been broken. */
+ c = utf8_to_unicode(str, MAX_MULTIBYTE_CHAR_LEN);
inline patch 2: src/common/wchar.c (+2/-1 lines) download inline-2.patch
diff --git a/src/common/wchar.c b/src/common/wchar.c
index a4bc29921de..c113cadf815 100644
--- a/src/common/wchar.c
+++ b/src/common/wchar.c
@@ -661,7 +661,8 @@ ucs_wcwidth(pg_wchar ucs)
static int
pg_utf_dsplen(const unsigned char *s)
{
- return ucs_wcwidth(utf8_to_unicode(s));
+ /* trust that input is not a truncated byte sequence */
+ return ucs_wcwidth(utf8_to_unicode(s, MAX_MULTIBYTE_CHAR_LEN));
}
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2025-12-15 20:23 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2025-12-15 20:23 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Sun, 2025-12-14 at 07:22 +0800, Chao Li wrote:
> This patch adds a length check to utf8_to_unicode(), I think which is
> where “safety” comes from. Can you please add an a little bit more to
> the commit message instead of only saying “improve safety”.
Right: it does not read past pg_mblen(), nor past the supplied length.
We could go further and check for valid continuation bytes, but the
other routines don't do that.
> It also deleted two redundant function declarations from pg_wchar.h,
> which may also worth a quick note in the commit message.
I committed that as an independent change and removed it from this
patch.
> + /* Assume byte sequence has not been broken. */
> + c = utf8_to_unicode(str, MAX_MULTIBYTE_CHAR_LEN);
> ```
>
> Here we need an empty line above the new comment.
Done, and I expanded the comment to explain why it's safe.
> pg_utf_dsplen(const unsigned char *s)
> {
> - return ucs_wcwidth(utf8_to_unicode(s));
> + /* trust that input is not a truncated byte sequence */
> + return ucs_wcwidth(utf8_to_unicode(s,
> MAX_MULTIBYTE_CHAR_LEN));
> }
> ```
>
> For the new comment, as a code reader, I wonder why we “trust” that?
We could use strlen(), but I was concerned that it might be used for
string fragments that aren't NUL-terminated, because it's intended for
a single character. A caller might reasonably assume that it wouldn't
read past pg_mblen().
So I changed the comment slightly to just say that it requires the
input is a valid UTF-8 sequence. Let me know if you have another
suggestion.
Regards,
Jeff Davis
Attachments:
v2-0001-Make-utf8_to_unicode-safer.patch (text/x-patch, 9.8K) download
From 94a80385eec23dbb85cc601063cac04e8d845a52 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Dec 2025 19:37:32 -0800
Subject: [PATCH v2] Make utf8_to_unicode() safer.
Require the input string length so that it doesn't read past the end
of a string if given an invalid (or truncated) byte sequence.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
contrib/fuzzystrmatch/daitch_mokotoff.c | 8 ++++++-
src/backend/utils/adt/pg_locale_builtin.c | 5 +++--
src/backend/utils/adt/varlena.c | 26 +++++++++++------------
src/common/saslprep.c | 4 +++-
src/common/unicode/case_test.c | 5 +++--
src/common/unicode_case.c | 12 +++++++----
src/common/wchar.c | 3 ++-
src/include/mb/pg_wchar.h | 12 +++++------
8 files changed, 43 insertions(+), 32 deletions(-)
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
index 07f895ae2bf..103a29d89d4 100644
--- a/contrib/fuzzystrmatch/daitch_mokotoff.c
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -401,7 +401,13 @@ read_char(const unsigned char *str, int *ix)
/* Decode UTF-8 character to ISO 10646 code point. */
str += *ix;
- c = utf8_to_unicode(str);
+
+ /*
+ * We assume the string does not contain truncated byte sequences because
+ * it came from pg_server_to_any(..., PG_UTF8), so we can safely use
+ * MAX_MULTIBYTE_CHAR_LEN.
+ */
+ c = utf8_to_unicode(str, MAX_MULTIBYTE_CHAR_LEN);
/* Advance *ix, but (for safety) not if we've reached end of string. */
if (c)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 0c2920112bb..28fc55278b0 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -59,12 +59,13 @@ static size_t
initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
+ unsigned char *str = (unsigned char *) wbstate->str;
while (wbstate->offset < wbstate->len &&
wbstate->str[wbstate->offset] != '\0')
{
- char32_t u = utf8_to_unicode((unsigned char *) wbstate->str +
- wbstate->offset);
+ char32_t u = utf8_to_unicode(str + wbstate->offset,
+ wbstate->len - wbstate->offset);
bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index baa5b44ea8d..d684370900d 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5423,19 +5423,17 @@ Datum
unicode_assigned(PG_FUNCTION_ARGS)
{
text *input = PG_GETARG_TEXT_PP(0);
- unsigned char *p;
- int size;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
if (GetDatabaseEncoding() != PG_UTF8)
ereport(ERROR,
(errmsg("Unicode categorization can only be performed if server encoding is UTF8")));
/* convert to char32_t */
- size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
- p = (unsigned char *) VARDATA_ANY(input);
- for (int i = 0; i < size; i++)
+ while (p < p_end)
{
- char32_t uchar = utf8_to_unicode(p);
+ char32_t uchar = utf8_to_unicode(p, p_end - p);
int category = unicode_category(uchar);
if (category == PG_U_UNASSIGNED)
@@ -5456,7 +5454,8 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
int size;
char32_t *input_chars;
char32_t *output_chars;
- unsigned char *p;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
text *result;
int i;
@@ -5465,14 +5464,13 @@ unicode_normalize_func(PG_FUNCTION_ARGS)
/* convert to char32_t */
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
input_chars = palloc((size + 1) * sizeof(char32_t));
- p = (unsigned char *) VARDATA_ANY(input);
for (i = 0; i < size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert(p == p_end);
/* action */
output_chars = unicode_normalize(form, input_chars);
@@ -5522,7 +5520,8 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
int size;
char32_t *input_chars;
char32_t *output_chars;
- unsigned char *p;
+ unsigned char *p = (unsigned char *) VARDATA_ANY(input);
+ unsigned char *p_end = p + VARSIZE_ANY_EXHDR(input);
int i;
UnicodeNormalizationQC quickcheck;
int output_size;
@@ -5533,14 +5532,13 @@ unicode_is_normalized(PG_FUNCTION_ARGS)
/* convert to char32_t */
size = pg_mbstrlen_with_len(VARDATA_ANY(input), VARSIZE_ANY_EXHDR(input));
input_chars = palloc((size + 1) * sizeof(char32_t));
- p = (unsigned char *) VARDATA_ANY(input);
for (i = 0; i < size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
- Assert((char *) p == VARDATA_ANY(input) + VARSIZE_ANY_EXHDR(input));
+ Assert(p == p_end);
/* quick check (see UAX #15) */
quickcheck = unicode_is_normalized_quickcheck(form, input_chars);
diff --git a/src/common/saslprep.c b/src/common/saslprep.c
index 101e8d65a4d..083702654b0 100644
--- a/src/common/saslprep.c
+++ b/src/common/saslprep.c
@@ -1055,6 +1055,7 @@ pg_saslprep(const char *input, char **output)
int i;
bool contains_RandALCat;
unsigned char *p;
+ unsigned char *p_end;
char32_t *wp;
/* Ensure we return *output as NULL on failure */
@@ -1088,9 +1089,10 @@ pg_saslprep(const char *input, char **output)
goto oom;
p = (unsigned char *) input;
+ p_end = p + strlen(input);
for (i = 0; i < input_size; i++)
{
- input_chars[i] = utf8_to_unicode(p);
+ input_chars[i] = utf8_to_unicode(p, p_end - p);
p += pg_utf_mblen(p);
}
input_chars[i] = (char32_t) '\0';
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 00d4f85e5a5..e63ce2fbeb9 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -51,12 +51,13 @@ static size_t
initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
+ unsigned char *str = (unsigned char *) wbstate->str;
while (wbstate->offset < wbstate->len &&
wbstate->str[wbstate->offset] != '\0')
{
- char32_t u = utf8_to_unicode((unsigned char *) wbstate->str +
- wbstate->offset);
+ char32_t u = utf8_to_unicode(str + wbstate->offset,
+ wbstate->len - wbstate->offset);
bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index e5e494db43c..a760094104c 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -223,15 +223,19 @@ convert_case(char *dst, size_t dstsize, const char *src, ssize_t srclen,
Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
(str_casekind != CaseTitle && !wbnext && !wbstate));
+ if (srclen < 0)
+ srclen = strlen(src);
+
if (str_casekind == CaseTitle)
{
boundary = wbnext(wbstate);
Assert(boundary == 0); /* start of text is always a boundary */
}
- while ((srclen < 0 || srcoff < srclen) && src[srcoff] != '\0')
+ while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((unsigned char *) src + srcoff);
+ char32_t u1 = utf8_to_unicode((unsigned char *) src + srcoff,
+ srclen - srcoff);
int u1len = unicode_utf8len(u1);
char32_t simple = 0;
const char32_t *special = NULL;
@@ -320,7 +324,7 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ char32_t curr = utf8_to_unicode(str + i, len - i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -344,7 +348,7 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ char32_t curr = utf8_to_unicode(str + i, len - i);
if (pg_u_prop_case_ignorable(curr))
continue;
diff --git a/src/common/wchar.c b/src/common/wchar.c
index a4bc29921de..a104234ae92 100644
--- a/src/common/wchar.c
+++ b/src/common/wchar.c
@@ -661,7 +661,8 @@ ucs_wcwidth(pg_wchar ucs)
static int
pg_utf_dsplen(const unsigned char *s)
{
- return ucs_wcwidth(utf8_to_unicode(s));
+ /* input must be a valid UTF-8 sequence */
+ return ucs_wcwidth(utf8_to_unicode(s, MAX_MULTIBYTE_CHAR_LEN));
}
/*
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index 5df67ceea87..6dc0fff332f 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -558,22 +558,20 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
- *
- * No error checks here, c must point to a long-enough string.
*/
static inline char32_t
-utf8_to_unicode(const unsigned char *c)
+utf8_to_unicode(const unsigned char *c, size_t len)
{
- if ((*c & 0x80) == 0)
+ if ((*c & 0x80) == 0 && len >= 1)
return (char32_t) c[0];
- else if ((*c & 0xe0) == 0xc0)
+ else if ((*c & 0xe0) == 0xc0 && len >= 2)
return (char32_t) (((c[0] & 0x1f) << 6) |
(c[1] & 0x3f));
- else if ((*c & 0xf0) == 0xe0)
+ else if ((*c & 0xf0) == 0xe0 && len >= 3)
return (char32_t) (((c[0] & 0x0f) << 12) |
((c[1] & 0x3f) << 6) |
(c[2] & 0x3f));
- else if ((*c & 0xf8) == 0xf0)
+ else if ((*c & 0xf8) == 0xf0 && len >= 4)
return (char32_t) (((c[0] & 0x07) << 18) |
((c[1] & 0x3f) << 12) |
((c[2] & 0x3f) << 6) |
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2025-12-15 23:34 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Chao Li @ 2025-12-15 23:34 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Dec 16, 2025, at 04:23, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Sun, 2025-12-14 at 07:22 +0800, Chao Li wrote:
>> This patch adds a length check to utf8_to_unicode(), I think which is
>> where “safety” comes from. Can you please add an a little bit more to
>> the commit message instead of only saying “improve safety”.
>
> Right: it does not read past pg_mblen(), nor past the supplied length.
>
> We could go further and check for valid continuation bytes, but the
> other routines don't do that.
>
>> It also deleted two redundant function declarations from pg_wchar.h,
>> which may also worth a quick note in the commit message.
>
> I committed that as an independent change and removed it from this
> patch.
>
>> + /* Assume byte sequence has not been broken. */
>> + c = utf8_to_unicode(str, MAX_MULTIBYTE_CHAR_LEN);
>> ```
>>
>> Here we need an empty line above the new comment.
>
> Done, and I expanded the comment to explain why it's safe.
>
>> pg_utf_dsplen(const unsigned char *s)
>> {
>> - return ucs_wcwidth(utf8_to_unicode(s));
>> + /* trust that input is not a truncated byte sequence */
>> + return ucs_wcwidth(utf8_to_unicode(s,
>> MAX_MULTIBYTE_CHAR_LEN));
>> }
>> ```
>>
>> For the new comment, as a code reader, I wonder why we “trust” that?
>
> We could use strlen(), but I was concerned that it might be used for
> string fragments that aren't NUL-terminated, because it's intended for
> a single character. A caller might reasonably assume that it wouldn't
> read past pg_mblen().
>
> So I changed the comment slightly to just say that it requires the
> input is a valid UTF-8 sequence. Let me know if you have another
> suggestion.
>
> Regards,
> Jeff Davis
>
>
> <v2-0001-Make-utf8_to_unicode-safer.patch>
V2 LGTM.
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2025-12-17 19:37 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2025-12-17 19:37 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Tue, 2025-12-16 at 07:34 +0800, Chao Li wrote:
> > <v2-0001-Make-utf8_to_unicode-safer.patch>
>
> V2 LGTM.
On second thought, if we're going to change something here, we should
probably have a more flexible API for both utf8_to_unicode() and
unicode_to_utf8().
Looking at the callers, I think we want to have signatures something
like:
/* returns number of bytes consumed, or -1 */
static inline ssize_t
utf8_to_unicode(char32_t *cp, const unsigned char *src, size_t srclen)
{
...
}
/* returns number of bytes written, or -1 */
static inline ssize_t
unicode_to_utf8(unsigned char *dst, size_t dstsize, char32_t cp)
{
...
}
That would make both APIs safer, and the caller wouldn't need to call
unicode_utf8len() or pg_utf8_mblen() separately.
We could also do more validation, but of course then the callers would
need to do something if they encounter a failure. We could also try to
catch NUL terminators in the middle of a sequence, which might be
useful.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-19 23:22 Jeff Davis <pgsql@j-davis.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-06-19 23:22 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Wed, 2025-12-17 at 11:37 -0800, Jeff Davis wrote:
> On Tue, 2025-12-16 at 07:34 +0800, Chao Li wrote:
> > > <v2-0001-Make-utf8_to_unicode-safer.patch>
> >
> > V2 LGTM.
>
> On second thought, if we're going to change something here, we should
> probably have a more flexible API for both utf8_to_unicode() and
> unicode_to_utf8().
New series:
0001: validates UTF8 before calling into unicode_case.c. Extra defense,
and simple to backport, but regresses performance of those functions.
It also might risk errors if somehow there is invalid UTF8.
0002: refactors to create an error path from unicode_case.c into
pg_locale_builtin.c, where a proper error can be thrown. This wins back
the performance lost in the previous commit. This is perhaps
backportable, but technically it changes an exported function
signature, so carries some very low risk.
0003: Adds utf8encode() and utf8decode(), which are iteration-friendly
and inlinable, and fully-validate UTF8 (e.g. rejects surrogate halves).
This is an enhancement so should not be backported.
0004: Make use of new API from unicode_case.c.
Regards,
Jeff Davis
Attachments: download all (4 files, .tar.gz)
v3-0001-unicode_case.c-ensure-valid-UTF8.patch (text/x-patch, 1.8K) download
From 4d59a316a147d68ff0113cec6b969037d2ee169e Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:09:31 -0700
Subject: [PATCH v3 1/4] unicode_case.c: ensure valid UTF8.
Should be valid, but check before calling unicode_strlower(), etc.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Backpatch-through: 17
---
src/backend/utils/adt/pg_locale_builtin.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 01d4f55b07e..7f167e751ea 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -86,6 +86,7 @@ static size_t
strlower_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
+ pg_verifymbstr(src, srclen, false);
return unicode_strlower(dest, destsize, src, srclen,
locale->builtin.casemap_full);
}
@@ -103,6 +104,7 @@ strtitle_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
.prev_alnum = false,
};
+ pg_verifymbstr(src, srclen, false);
return unicode_strtitle(dest, destsize, src, srclen,
locale->builtin.casemap_full,
initcap_wbnext, &wbstate);
@@ -112,6 +114,7 @@ static size_t
strupper_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
+ pg_verifymbstr(src, srclen, false);
return unicode_strupper(dest, destsize, src, srclen,
locale->builtin.casemap_full);
}
@@ -120,6 +123,7 @@ static size_t
strfold_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
+ pg_verifymbstr(src, srclen, false);
return unicode_strfold(dest, destsize, src, srclen,
locale->builtin.casemap_full);
}
--
2.43.0
v3-0002-Move-UTF8-checks-into-unicode_case.c.patch (text/x-patch, 15.0K) download
From fffa7153f563a19663a02e44196f377b83bf217f Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 16 Apr 2026 14:56:11 -0700
Subject: [PATCH v3 2/4] Move UTF8 checks into unicode_case.c.
Pre-checking UTF-8 is inefficient. Refactor the error paths so we can
catch UTF-8 errors while iterating, and return back to
pg_locale_builtin.c where we can throw the error.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 85 +++++++++++++++------
src/common/unicode/case_test.c | 33 +++++---
src/common/unicode_case.c | 92 ++++++++++++++++-------
src/include/common/unicode_case.h | 8 +-
4 files changed, 156 insertions(+), 62 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 7f167e751ea..96da9c6fcf3 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,21 +62,32 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ char32_t u;
+ bool curr_alnum;
+
+ if (wbstate->offset + ulen > wbstate->len)
+ {
+ wbstate->offset = wbstate->len;
+ return wbstate->len;
+ }
+
+ u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ wbstate->offset);
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
{
size_t prev_offset = wbstate->offset;
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
@@ -86,9 +97,16 @@ static size_t
strlower_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- pg_verifymbstr(src, srclen, false);
- return unicode_strlower(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strlower(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
@@ -96,36 +114,57 @@ strtitle_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
struct WordBoundaryState wbstate = {
- .str = src,
- .len = srclen,
- .offset = 0,
- .posix = !locale->builtin.casemap_full,
- .init = false,
- .prev_alnum = false,
+ .str = src,
+ .len = srclen,
+ .offset = 0,
+ .posix = !locale->builtin.casemap_full,
+ .init = false,
+ .prev_alnum = false,
};
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strtitle(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full,
+ initcap_wbnext, &wbstate);
- pg_verifymbstr(src, srclen, false);
- return unicode_strtitle(dest, destsize, src, srclen,
- locale->builtin.casemap_full,
- initcap_wbnext, &wbstate);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strupper_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- pg_verifymbstr(src, srclen, false);
- return unicode_strupper(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strupper(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strfold_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- pg_verifymbstr(src, srclen, false);
- return unicode_strfold(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strfold(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static bool
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index a0dbf00b671..ae0f86ffa0c 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -115,6 +115,7 @@ icu_test_full(char *str)
char icu_fold[BUFSZ];
UErrorCode status;
size_t len = strlen(str);
+ size_t consumed;
/* full case mapping doesn't use posix semantics */
struct WordBoundaryState wbstate = {
@@ -126,10 +127,10 @@ icu_test_full(char *str)
.prev_alnum = false,
};
- unicode_strlower(lower, BUFSZ, str, len, true);
- unicode_strtitle(title, BUFSZ, str, len, true, initcap_wbnext, &wbstate);
- unicode_strupper(upper, BUFSZ, str, len, true);
- unicode_strfold(fold, BUFSZ, str, len, true);
+ unicode_strlower(lower, BUFSZ, str, len, &consumed, true);
+ unicode_strtitle(title, BUFSZ, str, len, &consumed, true, initcap_wbnext, &wbstate);
+ unicode_strupper(upper, BUFSZ, str, len, &consumed, true);
+ unicode_strfold(fold, BUFSZ, str, len, &consumed, true);
status = U_ZERO_ERROR;
ucasemap_utf8ToLower(casemap, icu_lower, BUFSZ, str, len, &status);
status = U_ZERO_ERROR;
@@ -260,13 +261,16 @@ static size_t
tfunc_lower(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strlower(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strlower(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_title(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
+ size_t consumed;
struct WordBoundaryState wbstate = {
.str = src,
.len = srclen,
@@ -275,27 +279,34 @@ tfunc_title(char *dst, size_t dstsize, const char *src,
.prev_alnum = false,
};
- return unicode_strtitle(dst, dstsize, src, srclen, true, initcap_wbnext,
- &wbstate);
+ return unicode_strtitle(dst, dstsize, src, srclen, &consumed, true,
+ initcap_wbnext, &wbstate);
}
static size_t
tfunc_upper(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strupper(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strupper(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_fold(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strfold(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strfold(dst, dstsize, src, srclen, &consumed, true);
}
static void
test_convert_case(void)
{
+ size_t needed;
+ size_t consumed;
+
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
/* test adjust-to-cased behavior */
@@ -320,6 +331,10 @@ test_convert_case(void)
/* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
+ /* invalid UTF8 */
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, &consumed, false);
+ Assert(consumed == 3);
+ Assert(needed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index d6ee00b7d9c..4a692cfa249 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -40,8 +40,8 @@ static const char32_t *const casekind_map[NCaseKind] =
static char32_t find_case_map(char32_t ucs, const char32_t *map);
static size_t convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate);
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate);
static enum CaseMapResult casemap(char32_t u1, CaseKind casekind, bool full,
const char *src, size_t srclen, size_t srcoff,
char32_t *simple, const char32_t **special);
@@ -82,7 +82,8 @@ unicode_casefold_simple(char32_t code)
* unicode_strlower()
*
* Convert src to lowercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -98,17 +99,18 @@ unicode_casefold_simple(char32_t code)
*/
size_t
unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseLower, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseLower, full,
+ NULL, NULL);
}
/*
* unicode_strtitle()
*
* Convert src to titlecase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -134,17 +136,19 @@ unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full, WordBoundaryNext wbnext, void *wbstate)
+ size_t *pconsumed, bool full, WordBoundaryNext wbnext,
+ void *wbstate)
{
- return convert_case(dst, dstsize, src, srclen, CaseTitle, full, wbnext,
- wbstate);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseTitle, full,
+ wbnext, wbstate);
}
/*
* unicode_strupper()
*
* Convert src to uppercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -160,17 +164,18 @@ unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseUpper, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseUpper, full,
+ NULL, NULL);
}
/*
* unicode_strfold()
*
* Case fold src, and return the result length (not including terminating
- * NUL).
+ * NUL). Sets *pconsumed to the amount of src successfully consumed; if less
+ * than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -183,10 +188,26 @@ unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseFold, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseFold, full,
+ NULL, NULL);
+}
+
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+ if ((*s & 0x80) == 0)
+ return 1;
+ else if ((*s & 0xe0) == 0xc0)
+ return 2;
+ else if ((*s & 0xf0) == 0xe0)
+ return 3;
+ else if ((*s & 0xf8) == 0xf0)
+ return 4;
+ else
+ return -1;
}
/*
@@ -207,8 +228,8 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
static size_t
convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate)
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate)
{
/* character CaseKind varies while titlecasing */
CaseKind chr_casekind = str_casekind;
@@ -227,12 +248,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
- int u1len = unicode_utf8len(u1);
+ int u1len = utf8_mblen((const unsigned char *) src + srcoff);
+ char32_t u1;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ /* invalid UTF8 */
+ if (u1len < 0 || srcoff + u1len > srclen)
+ break;
+
+ u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -293,6 +320,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
if (result_len < dstsize)
dst[result_len] = '\0';
+ *pconsumed = srcoff;
return result_len;
}
@@ -316,7 +344,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -327,8 +362,6 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
}
/* end of string is not followed by a Cased character */
@@ -340,7 +373,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
diff --git a/src/include/common/unicode_case.h b/src/include/common/unicode_case.h
index 03add78cabe..1cbc0c14bc2 100644
--- a/src/include/common/unicode_case.h
+++ b/src/include/common/unicode_case.h
@@ -21,13 +21,13 @@ char32_t unicode_titlecase_simple(char32_t code);
char32_t unicode_uppercase_simple(char32_t code);
char32_t unicode_casefold_simple(char32_t code);
size_t unicode_strlower(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strtitle(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full,
+ size_t srclen, size_t *pconsumed, bool full,
WordBoundaryNext wbnext, void *wbstate);
size_t unicode_strupper(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strfold(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
#endif /* UNICODE_CASE_H */
--
2.43.0
v3-0003-Validating-iterator-friendly-UTF8-encoder-decoder.patch (text/x-patch, 5.3K) download
From ba582fcb653d27110675831f29b3b088609c02ff Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v3 3/4] Validating, iterator-friendly UTF8 encoder/decoder
API.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/include/mb/pg_wchar.h | 160 +++++++++++++++++++++++++++++++++++++-
1 file changed, 158 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..d8ea77c3fe0 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,161 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if (nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+
+ if ((*src & 0xe0) == 0xc0)
+ nbytes = 2;
+ else if ((*src & 0xf0) == 0xe0)
+ nbytes = 3;
+ else if ((*src & 0xf8) == 0xf0)
+ nbytes = 4;
+ else
+ goto invalid;
+
+ /* truncated */
+ if (srclen < nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ if (nbytes == 2)
+ {
+ /* check continuation byte */
+ if ((src[1] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x1f) << 6) |
+ (src[1] & 0x3f));
+
+ /* overlong */
+ if (codepoint < 0x0080)
+ goto invalid;
+ }
+ else if (nbytes == 3)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x0f) << 12) |
+ ((src[1] & 0x3f) << 6) |
+ (src[2] & 0x3f));
+
+ /* overlong or surrogate half */
+ if (codepoint < 0x0800 ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+ }
+ else if (nbytes == 4)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80 ||
+ (src[3] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x07) << 18) |
+ ((src[1] & 0x3f) << 12) |
+ ((src[2] & 0x3f) << 6) |
+ (src[3] & 0x3f));
+
+ /* overlong or out-of-range */
+ if (codepoint < 0x10000 || codepoint > 0x10FFFF)
+ goto invalid;
+ }
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +571,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
v3-0004-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch (text/x-patch, 6.4K) download
From 942f374a4383502cbb66d0d83f0ba317962a9a24 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v3 4/4] unicode_case.c: use new utf8encode/utf8decode APIs.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 17 +++---
src/common/unicode/case_test.c | 2 +-
src/common/unicode_case.c | 70 +++++++++++------------
3 files changed, 44 insertions(+), 45 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 96da9c6fcf3..a826ec1cfa8 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,25 +62,26 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (wbstate->offset + ulen > wbstate->len)
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
+ /* invalid UTF8 */
+ if (ulen <= 0)
{
+ wbstate->init = true;
wbstate->offset = wbstate->len;
- return wbstate->len;
+ return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
{
- size_t prev_offset = wbstate->offset;
-
wbstate->init = true;
wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index ae0f86ffa0c..71a6f2bbe70 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -179,7 +179,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 4a692cfa249..d89f5ca4740 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
result_len += u2len;
}
@@ -304,10 +297,17 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
+
+ u2len = utf8encode(p, remaining, u2);
result_len += u2len;
}
@@ -344,15 +344,15 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- int u1len = utf8_mblen((const unsigned char *) str + i);
+ int u1len;
char32_t curr;
+ u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i);
+
/* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
+ if (u1len <= 0)
return false;
- curr = utf8_to_unicode(str + i);
-
if (pg_u_prop_case_ignorable(curr))
continue;
else if (pg_u_prop_cased(curr))
@@ -373,15 +373,15 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- int u1len = utf8_mblen((const unsigned char *) str + i);
+ int u1len;
char32_t curr;
+ u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i);
+
/* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
+ if (u1len <= 0)
return false;
- curr = utf8_to_unicode(str + i);
-
if (pg_u_prop_case_ignorable(curr))
continue;
else if (pg_u_prop_cased(curr))
@@ -391,8 +391,6 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
}
return true;
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-23 02:02 Jeff Davis <pgsql@j-davis.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-06-23 02:02 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Fri, 2026-06-19 at 16:22 -0700, Jeff Davis wrote:
> On Wed, 2025-12-17 at 11:37 -0800, Jeff Davis wrote:
> > On Tue, 2025-12-16 at 07:34 +0800, Chao Li wrote:
> > > > <v2-0001-Make-utf8_to_unicode-safer.patch>
> > >
> > > V2 LGTM.
> >
> > On second thought, if we're going to change something here, we
> > should
> > probably have a more flexible API for both utf8_to_unicode() and
> > unicode_to_utf8().
v4 attached.
The main difference is that the first patch is more backportable. For
backbranches, I think the safest thing if we encounter invalid UTF8 is
to just terminate and return early. In master, we can change the API to
properly return the error upward.
Performance is not affected much, though in my brief tests it appeared
that 0002 lost a bit and then 0004 gained it back. But we gain full
UTF8 validation and safer UTF8 iterator APIs.
Regards,
Jeff Davis
Attachments: download all (4 files, .tar.gz)
v4-0001-unicode_case.c-defend-against-invalid-UTF8.patch (text/x-patch, 4.6K) download
From 3ccf14e0d6234ab4fcf24cf56f53f512649bc658 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:30:01 -0700
Subject: [PATCH v4 1/4] unicode_case.c: defend against invalid UTF8.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 17
---
src/backend/utils/adt/pg_locale_builtin.c | 24 ++++++++---
src/common/unicode_case.c | 52 +++++++++++++++++++----
2 files changed, 62 insertions(+), 14 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 01d4f55b07e..7c36fd5091b 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,21 +62,33 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ /* invalid UTF8 */
+ if (wbstate->offset + ulen > wbstate->len)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+ u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ wbstate->offset);
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index d6ee00b7d9c..42eb7d22211 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -189,6 +189,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL);
}
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+ if ((*s & 0x80) == 0)
+ return 1;
+ else if ((*s & 0xe0) == 0xc0)
+ return 2;
+ else if ((*s & 0xf0) == 0xe0)
+ return 3;
+ else if ((*s & 0xf8) == 0xf0)
+ return 4;
+ else
+ return -1;
+}
+
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -227,12 +243,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
- int u1len = unicode_utf8len(u1);
+ int u1len = utf8_mblen((const unsigned char *) src + srcoff);
+ char32_t u1;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ /* invalid UTF8 */
+ if (u1len < 0 || srcoff + u1len > srclen)
+ break;
+
+ u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -316,7 +338,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -327,8 +356,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
/* end of string is not followed by a Cased character */
@@ -340,7 +369,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -351,8 +387,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
return true;
--
2.43.0
v4-0002-unicode_case.c-change-API-to-signal-UTF8-decoding.patch (text/x-patch, 11.5K) download
From d3621a42037e91a02b24c8974bd48ce9a42febf4 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:31:38 -0700
Subject: [PATCH v4 2/4] unicode_case.c: change API to signal UTF8 decoding
error.
Errors at this point are not expected, but if encountered, signal to
the caller so it can raise the appropriate error.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 50 +++++++++++++++++++----
src/common/unicode/case_test.c | 33 +++++++++++----
src/common/unicode_case.c | 46 ++++++++++++---------
src/include/common/unicode_case.h | 8 ++--
4 files changed, 95 insertions(+), 42 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 7c36fd5091b..5619daf43c3 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -98,8 +98,16 @@ static size_t
strlower_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strlower(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strlower(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
@@ -114,26 +122,50 @@ strtitle_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
.init = false,
.prev_alnum = false,
};
+ size_t consumed;
+ size_t result;
- return unicode_strtitle(dest, destsize, src, srclen,
- locale->builtin.casemap_full,
- initcap_wbnext, &wbstate);
+ result = unicode_strtitle(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full,
+ initcap_wbnext, &wbstate);
+
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strupper_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strupper(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strupper(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strfold_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strfold(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strfold(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static bool
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index a0dbf00b671..ae0f86ffa0c 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -115,6 +115,7 @@ icu_test_full(char *str)
char icu_fold[BUFSZ];
UErrorCode status;
size_t len = strlen(str);
+ size_t consumed;
/* full case mapping doesn't use posix semantics */
struct WordBoundaryState wbstate = {
@@ -126,10 +127,10 @@ icu_test_full(char *str)
.prev_alnum = false,
};
- unicode_strlower(lower, BUFSZ, str, len, true);
- unicode_strtitle(title, BUFSZ, str, len, true, initcap_wbnext, &wbstate);
- unicode_strupper(upper, BUFSZ, str, len, true);
- unicode_strfold(fold, BUFSZ, str, len, true);
+ unicode_strlower(lower, BUFSZ, str, len, &consumed, true);
+ unicode_strtitle(title, BUFSZ, str, len, &consumed, true, initcap_wbnext, &wbstate);
+ unicode_strupper(upper, BUFSZ, str, len, &consumed, true);
+ unicode_strfold(fold, BUFSZ, str, len, &consumed, true);
status = U_ZERO_ERROR;
ucasemap_utf8ToLower(casemap, icu_lower, BUFSZ, str, len, &status);
status = U_ZERO_ERROR;
@@ -260,13 +261,16 @@ static size_t
tfunc_lower(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strlower(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strlower(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_title(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
+ size_t consumed;
struct WordBoundaryState wbstate = {
.str = src,
.len = srclen,
@@ -275,27 +279,34 @@ tfunc_title(char *dst, size_t dstsize, const char *src,
.prev_alnum = false,
};
- return unicode_strtitle(dst, dstsize, src, srclen, true, initcap_wbnext,
- &wbstate);
+ return unicode_strtitle(dst, dstsize, src, srclen, &consumed, true,
+ initcap_wbnext, &wbstate);
}
static size_t
tfunc_upper(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strupper(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strupper(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_fold(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strfold(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strfold(dst, dstsize, src, srclen, &consumed, true);
}
static void
test_convert_case(void)
{
+ size_t needed;
+ size_t consumed;
+
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
/* test adjust-to-cased behavior */
@@ -320,6 +331,10 @@ test_convert_case(void)
/* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
+ /* invalid UTF8 */
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, &consumed, false);
+ Assert(consumed == 3);
+ Assert(needed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 42eb7d22211..79f8c7f14a7 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -40,8 +40,8 @@ static const char32_t *const casekind_map[NCaseKind] =
static char32_t find_case_map(char32_t ucs, const char32_t *map);
static size_t convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate);
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate);
static enum CaseMapResult casemap(char32_t u1, CaseKind casekind, bool full,
const char *src, size_t srclen, size_t srcoff,
char32_t *simple, const char32_t **special);
@@ -82,7 +82,8 @@ unicode_casefold_simple(char32_t code)
* unicode_strlower()
*
* Convert src to lowercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -98,17 +99,18 @@ unicode_casefold_simple(char32_t code)
*/
size_t
unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseLower, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseLower, full,
+ NULL, NULL);
}
/*
* unicode_strtitle()
*
* Convert src to titlecase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -134,17 +136,19 @@ unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full, WordBoundaryNext wbnext, void *wbstate)
+ size_t *pconsumed, bool full, WordBoundaryNext wbnext,
+ void *wbstate)
{
- return convert_case(dst, dstsize, src, srclen, CaseTitle, full, wbnext,
- wbstate);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseTitle, full,
+ wbnext, wbstate);
}
/*
* unicode_strupper()
*
* Convert src to uppercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -160,17 +164,18 @@ unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseUpper, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseUpper, full,
+ NULL, NULL);
}
/*
* unicode_strfold()
*
* Case fold src, and return the result length (not including terminating
- * NUL).
+ * NUL). Sets *pconsumed to the amount of src successfully consumed; if less
+ * than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -183,10 +188,10 @@ unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseFold, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseFold, full,
+ NULL, NULL);
}
/* local version of pg_utf_mblen() to be inlinable */
@@ -223,8 +228,8 @@ utf8_mblen(const unsigned char *s)
*/
static size_t
convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate)
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate)
{
/* character CaseKind varies while titlecasing */
CaseKind chr_casekind = str_casekind;
@@ -315,6 +320,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
if (result_len < dstsize)
dst[result_len] = '\0';
+ *pconsumed = srcoff;
return result_len;
}
diff --git a/src/include/common/unicode_case.h b/src/include/common/unicode_case.h
index 03add78cabe..1cbc0c14bc2 100644
--- a/src/include/common/unicode_case.h
+++ b/src/include/common/unicode_case.h
@@ -21,13 +21,13 @@ char32_t unicode_titlecase_simple(char32_t code);
char32_t unicode_uppercase_simple(char32_t code);
char32_t unicode_casefold_simple(char32_t code);
size_t unicode_strlower(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strtitle(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full,
+ size_t srclen, size_t *pconsumed, bool full,
WordBoundaryNext wbnext, void *wbstate);
size_t unicode_strupper(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strfold(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
#endif /* UNICODE_CASE_H */
--
2.43.0
v4-0003-Validating-iterator-friendly-UTF8-encoder-decoder.patch (text/x-patch, 5.3K) download
From 3ef15be4a2ad8e99d9f8b6d757c17044166c5b8d Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v4 3/4] Validating, iterator-friendly UTF8 encoder/decoder
API.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/include/mb/pg_wchar.h | 160 +++++++++++++++++++++++++++++++++++++-
1 file changed, 158 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..d8ea77c3fe0 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,161 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if (nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+
+ if ((*src & 0xe0) == 0xc0)
+ nbytes = 2;
+ else if ((*src & 0xf0) == 0xe0)
+ nbytes = 3;
+ else if ((*src & 0xf8) == 0xf0)
+ nbytes = 4;
+ else
+ goto invalid;
+
+ /* truncated */
+ if (srclen < nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ if (nbytes == 2)
+ {
+ /* check continuation byte */
+ if ((src[1] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x1f) << 6) |
+ (src[1] & 0x3f));
+
+ /* overlong */
+ if (codepoint < 0x0080)
+ goto invalid;
+ }
+ else if (nbytes == 3)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x0f) << 12) |
+ ((src[1] & 0x3f) << 6) |
+ (src[2] & 0x3f));
+
+ /* overlong or surrogate half */
+ if (codepoint < 0x0800 ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+ }
+ else if (nbytes == 4)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80 ||
+ (src[3] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x07) << 18) |
+ ((src[1] & 0x3f) << 12) |
+ ((src[2] & 0x3f) << 6) |
+ (src[3] & 0x3f));
+
+ /* overlong or out-of-range */
+ if (codepoint < 0x10000 || codepoint > 0x10FFFF)
+ goto invalid;
+ }
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +571,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
v4-0004-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch (text/x-patch, 6.2K) download
From 3b7ebe21b1df897579832690eaecf9cf1041328f Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v4 4/4] unicode_case.c: use new utf8encode/utf8decode APIs.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 10 ++--
src/common/unicode/case_test.c | 2 +-
src/common/unicode_case.c | 72 ++++++++++++-----------
3 files changed, 43 insertions(+), 41 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..63516e7174f 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,22 +62,22 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
size_t prev_offset = wbstate->offset;
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
/* invalid UTF8 */
- if (wbstate->offset + ulen > wbstate->len)
+ if (ulen <= 0)
{
wbstate->init = true;
wbstate->offset = wbstate->len;
return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index ae0f86ffa0c..71a6f2bbe70 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -179,7 +179,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 79f8c7f14a7..26833cf0c53 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -304,10 +298,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -344,15 +346,15 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- int u1len = utf8_mblen((const unsigned char *) str + i);
+ int u1len;
char32_t curr;
+ u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i);
+
/* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
+ if (u1len <= 0)
return false;
- curr = utf8_to_unicode(str + i);
-
if (pg_u_prop_case_ignorable(curr))
continue;
else if (pg_u_prop_cased(curr))
@@ -371,19 +373,19 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
return true;
/* iterate forwards, looking for Cased character */
- for (int i = offset + 1; i < len && str[i] != '\0'; i++)
+ for (int i = offset + 1; i < len; i++)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- int u1len = utf8_mblen((const unsigned char *) str + i);
+ int u1len;
char32_t curr;
+ u1len = utf8decode(&curr, (const unsigned char *) str + i, len - i);
+
/* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
+ if (u1len <= 0)
return false;
- curr = utf8_to_unicode(str + i);
-
if (pg_u_prop_case_ignorable(curr))
continue;
else if (pg_u_prop_cased(curr))
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-24 05:45 Jeff Davis <pgsql@j-davis.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 2 replies; 27+ messages in thread
From: Jeff Davis @ 2026-06-24 05:45 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Mon, 2026-06-22 at 19:02 -0700, Jeff Davis wrote:
> v4 attached.
v5 attached.
There's an extra patch 0002 to fix a logic bug when handling final
sigma (only affects the builtin pg_unicode_fast locale), which I think
should be backported to 18.
Also added tests.
Regards,
Jeff Davis
Attachments: download all (5 files, .tar.gz)
v5-0001-unicode_case.c-defend-against-invalid-UTF8.patch (text/x-patch, 5.5K) download
From 9a9bfbc4f3866d77e48933df052086a2558e7263 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:30:01 -0700
Subject: [PATCH v5 1/5] unicode_case.c: defend against invalid UTF8.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 17
---
src/backend/utils/adt/pg_locale_builtin.c | 24 ++++++++---
src/common/unicode/case_test.c | 8 ++++
src/common/unicode_case.c | 52 +++++++++++++++++++----
3 files changed, 70 insertions(+), 14 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 01d4f55b07e..7c36fd5091b 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,21 +62,33 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ /* invalid UTF8 */
+ if (wbstate->offset + ulen > wbstate->len)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+ u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ wbstate->offset);
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index a0dbf00b671..31ea94513bf 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -296,6 +296,8 @@ tfunc_fold(char *dst, size_t dstsize, const char *src,
static void
test_convert_case(void)
{
+ size_t needed;
+
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
/* test adjust-to-cased behavior */
@@ -320,6 +322,12 @@ test_convert_case(void)
/* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
+ /* invalid UTF8: truncated multibyte sequence */
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
+ Assert(needed == 3);
+ /* invalid UTF8: invalid byte */
+ needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
+ Assert(needed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index d6ee00b7d9c..42eb7d22211 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -189,6 +189,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL);
}
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+ if ((*s & 0x80) == 0)
+ return 1;
+ else if ((*s & 0xe0) == 0xc0)
+ return 2;
+ else if ((*s & 0xf0) == 0xe0)
+ return 3;
+ else if ((*s & 0xf8) == 0xf0)
+ return 4;
+ else
+ return -1;
+}
+
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -227,12 +243,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
- int u1len = unicode_utf8len(u1);
+ int u1len = utf8_mblen((const unsigned char *) src + srcoff);
+ char32_t u1;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ /* invalid UTF8 */
+ if (u1len < 0 || srcoff + u1len > srclen)
+ break;
+
+ u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -316,7 +338,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -327,8 +356,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
/* end of string is not followed by a Cased character */
@@ -340,7 +369,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -351,8 +387,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
return true;
--
2.43.0
v5-0002-pg_unicode_fast-fix-final-sigma-logic.patch (text/x-patch, 5.3K) download
From c389e8f3d47c51183702f468846c0ad8c33beaae Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Tue, 23 Jun 2026 17:09:49 -0700
Subject: [PATCH v5 2/5] pg_unicode_fast: fix final sigma logic.
If the string is preceded only by Case Ignorable characters, don't
consider it to be a final sigma.
In the process, refactor so that the preceding and following
characters are found first, and then the rule is applied, to improve
clarity.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 18
---
src/common/unicode_case.c | 88 ++++++++++------------
src/test/regress/expected/collate.utf8.out | 6 ++
src/test/regress/sql/collate.utf8.sql | 1 +
3 files changed, 47 insertions(+), 48 deletions(-)
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 42eb7d22211..dd5b3ba86d0 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -323,75 +323,67 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
* 3-17. The character at the given offset must be directly preceded by a
* Cased character, and must not be directly followed by a Cased character.
*
- * Case_Ignorable characters are ignored. NB: some characters may be both
+ * Case_Ignorable characters are ignored. Neither beginning of string nor end
+ * of string are considered Cased characters. NB: some characters may be both
* Cased and Case_Ignorable, in which case they are ignored.
*/
static bool
check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
- /* the start of the string is not preceded by a Cased character */
- if (offset == 0)
- return false;
+ bool preceded_by_cased = false;
+ bool followed_by_cased = false;
+ char32_t curr;
+ int ulen;
- /* iterate backwards, looking for Cased character */
- for (int i = offset - 1; i >= 0; i--)
+ /* iterate backwards looking for preceding character */
+ for (int i = offset; i > 0;)
{
- if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
- {
- int u1len = utf8_mblen((const unsigned char *) str + i);
- char32_t curr;
+ /* skip backwards through continuation bytes */
+ i--;
+ if ((str[i] & 0xC0) == 0x80)
+ continue;
- /* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
- return false;
+ /* now at leading byte of previous sequence */
+ Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- curr = utf8_to_unicode(str + i);
+ ulen = utf8_mblen((const unsigned char *) str + i);
- if (pg_u_prop_case_ignorable(curr))
- continue;
- else if (pg_u_prop_cased(curr))
- break;
- else
- return false;
+ /* invalid UTF8 */
+ if (ulen < 0 || i + ulen > len)
+ return false;
+
+ curr = utf8_to_unicode((const unsigned char *) str + i);
+
+ if (!pg_u_prop_case_ignorable(curr))
+ {
+ preceded_by_cased = pg_u_prop_cased(curr);
+ break;
}
- else if ((str[i] & 0xC0) == 0x80)
- continue;
- else
- return false; /* invalid UTF8 */
}
- /* end of string is not followed by a Cased character */
- if (offset == len)
- return true;
+ ulen = utf8_mblen((const unsigned char *) str + offset);
- /* iterate forwards, looking for Cased character */
- for (int i = offset + 1; i < len && str[i] != '\0'; i++)
+ /* iterate forward looking for following character */
+ for (int i = offset + ulen; i < len;)
{
- if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
- {
- int u1len = utf8_mblen((const unsigned char *) str + i);
- char32_t curr;
+ ulen = utf8_mblen((const unsigned char *) str + i);
- /* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
- return false;
+ /* invalid UTF8 */
+ if (ulen < 0 || i + ulen > len)
+ return false;
- curr = utf8_to_unicode(str + i);
+ curr = utf8_to_unicode((const unsigned char *) str + i);
- if (pg_u_prop_case_ignorable(curr))
- continue;
- else if (pg_u_prop_cased(curr))
- return false;
- else
- break;
+ if (!pg_u_prop_case_ignorable(curr))
+ {
+ followed_by_cased = pg_u_prop_cased(curr);
+ break;
}
- else if ((str[i] & 0xC0) == 0x80)
- continue;
- else
- return false; /* invalid UTF8 */
+
+ i += ulen;
}
- return true;
+ return (preceded_by_cased && !followed_by_cased);
}
/*
diff --git a/src/test/regress/expected/collate.utf8.out b/src/test/regress/expected/collate.utf8.out
index 0c3ab5c89b2..99fdc111fa4 100644
--- a/src/test/regress/expected/collate.utf8.out
+++ b/src/test/regress/expected/collate.utf8.out
@@ -263,6 +263,12 @@ SELECT lower('ᾼΣͅΑ' COLLATE PG_UNICODE_FAST); -- 0391 0345 03A3 0345 0391
ᾳσͅα
(1 row)
+SELECT lower(U&'\0300\03A3' COLLATE PG_UNICODE_FAST);
+ lower
+-------
+ ̀σ
+(1 row)
+
-- properties
SELECT 'xyz' ~ '[[:alnum:]]' COLLATE PG_UNICODE_FAST;
?column?
diff --git a/src/test/regress/sql/collate.utf8.sql b/src/test/regress/sql/collate.utf8.sql
index d6d14220ab3..22aecee3a60 100644
--- a/src/test/regress/sql/collate.utf8.sql
+++ b/src/test/regress/sql/collate.utf8.sql
@@ -128,6 +128,7 @@ SELECT lower('0Σ' COLLATE PG_UNICODE_FAST); -- 0030 03A3
SELECT lower('ΑΣΑ' COLLATE PG_UNICODE_FAST); -- 0391 03A3 0391
SELECT lower('ἈΣ̓Α' COLLATE PG_UNICODE_FAST); -- 0391 0343 03A3 0343 0391
SELECT lower('ᾼΣͅΑ' COLLATE PG_UNICODE_FAST); -- 0391 0345 03A3 0345 0391
+SELECT lower(U&'\0300\03A3' COLLATE PG_UNICODE_FAST);
-- properties
--
2.43.0
v5-0003-unicode_case.c-change-API-to-signal-UTF8-decoding.patch (text/x-patch, 11.8K) download
From 7f2494e90f041cb6e118c0a6a062b9e5364bc799 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:31:38 -0700
Subject: [PATCH v5 3/5] unicode_case.c: change API to signal UTF8 decoding
error.
Errors at this point are not expected, but if encountered, signal to
the caller so it can raise the appropriate error.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 50 +++++++++++++++++++----
src/common/unicode/case_test.c | 37 ++++++++++-------
src/common/unicode_case.c | 46 ++++++++++++---------
src/include/common/unicode_case.h | 8 ++--
4 files changed, 94 insertions(+), 47 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 7c36fd5091b..5619daf43c3 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -98,8 +98,16 @@ static size_t
strlower_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strlower(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strlower(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
@@ -114,26 +122,50 @@ strtitle_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
.init = false,
.prev_alnum = false,
};
+ size_t consumed;
+ size_t result;
- return unicode_strtitle(dest, destsize, src, srclen,
- locale->builtin.casemap_full,
- initcap_wbnext, &wbstate);
+ result = unicode_strtitle(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full,
+ initcap_wbnext, &wbstate);
+
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strupper_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strupper(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strupper(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strfold_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strfold(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strfold(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static bool
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 31ea94513bf..08421d9e5ca 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -115,6 +115,7 @@ icu_test_full(char *str)
char icu_fold[BUFSZ];
UErrorCode status;
size_t len = strlen(str);
+ size_t consumed;
/* full case mapping doesn't use posix semantics */
struct WordBoundaryState wbstate = {
@@ -126,10 +127,10 @@ icu_test_full(char *str)
.prev_alnum = false,
};
- unicode_strlower(lower, BUFSZ, str, len, true);
- unicode_strtitle(title, BUFSZ, str, len, true, initcap_wbnext, &wbstate);
- unicode_strupper(upper, BUFSZ, str, len, true);
- unicode_strfold(fold, BUFSZ, str, len, true);
+ unicode_strlower(lower, BUFSZ, str, len, &consumed, true);
+ unicode_strtitle(title, BUFSZ, str, len, &consumed, true, initcap_wbnext, &wbstate);
+ unicode_strupper(upper, BUFSZ, str, len, &consumed, true);
+ unicode_strfold(fold, BUFSZ, str, len, &consumed, true);
status = U_ZERO_ERROR;
ucasemap_utf8ToLower(casemap, icu_lower, BUFSZ, str, len, &status);
status = U_ZERO_ERROR;
@@ -260,13 +261,16 @@ static size_t
tfunc_lower(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strlower(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strlower(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_title(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
+ size_t consumed;
struct WordBoundaryState wbstate = {
.str = src,
.len = srclen,
@@ -275,28 +279,33 @@ tfunc_title(char *dst, size_t dstsize, const char *src,
.prev_alnum = false,
};
- return unicode_strtitle(dst, dstsize, src, srclen, true, initcap_wbnext,
- &wbstate);
+ return unicode_strtitle(dst, dstsize, src, srclen, &consumed, true,
+ initcap_wbnext, &wbstate);
}
static size_t
tfunc_upper(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strupper(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strupper(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_fold(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strfold(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strfold(dst, dstsize, src, srclen, &consumed, true);
}
static void
test_convert_case(void)
{
size_t needed;
+ size_t consumed;
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
@@ -323,11 +332,11 @@ test_convert_case(void)
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
/* invalid UTF8: truncated multibyte sequence */
- needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
- Assert(needed == 3);
- /* invalid UTF8: invalid byte */
- needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
- Assert(needed == 3);
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: leading byte invalid length */
+ needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index dd5b3ba86d0..24753aaab09 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -40,8 +40,8 @@ static const char32_t *const casekind_map[NCaseKind] =
static char32_t find_case_map(char32_t ucs, const char32_t *map);
static size_t convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate);
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate);
static enum CaseMapResult casemap(char32_t u1, CaseKind casekind, bool full,
const char *src, size_t srclen, size_t srcoff,
char32_t *simple, const char32_t **special);
@@ -82,7 +82,8 @@ unicode_casefold_simple(char32_t code)
* unicode_strlower()
*
* Convert src to lowercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -98,17 +99,18 @@ unicode_casefold_simple(char32_t code)
*/
size_t
unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseLower, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseLower, full,
+ NULL, NULL);
}
/*
* unicode_strtitle()
*
* Convert src to titlecase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -134,17 +136,19 @@ unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full, WordBoundaryNext wbnext, void *wbstate)
+ size_t *pconsumed, bool full, WordBoundaryNext wbnext,
+ void *wbstate)
{
- return convert_case(dst, dstsize, src, srclen, CaseTitle, full, wbnext,
- wbstate);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseTitle, full,
+ wbnext, wbstate);
}
/*
* unicode_strupper()
*
* Convert src to uppercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -160,17 +164,18 @@ unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseUpper, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseUpper, full,
+ NULL, NULL);
}
/*
* unicode_strfold()
*
* Case fold src, and return the result length (not including terminating
- * NUL).
+ * NUL). Sets *pconsumed to the amount of src successfully consumed; if less
+ * than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -183,10 +188,10 @@ unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseFold, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseFold, full,
+ NULL, NULL);
}
/* local version of pg_utf_mblen() to be inlinable */
@@ -223,8 +228,8 @@ utf8_mblen(const unsigned char *s)
*/
static size_t
convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate)
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate)
{
/* character CaseKind varies while titlecasing */
CaseKind chr_casekind = str_casekind;
@@ -315,6 +320,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
if (result_len < dstsize)
dst[result_len] = '\0';
+ *pconsumed = srcoff;
return result_len;
}
diff --git a/src/include/common/unicode_case.h b/src/include/common/unicode_case.h
index 03add78cabe..1cbc0c14bc2 100644
--- a/src/include/common/unicode_case.h
+++ b/src/include/common/unicode_case.h
@@ -21,13 +21,13 @@ char32_t unicode_titlecase_simple(char32_t code);
char32_t unicode_uppercase_simple(char32_t code);
char32_t unicode_casefold_simple(char32_t code);
size_t unicode_strlower(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strtitle(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full,
+ size_t srclen, size_t *pconsumed, bool full,
WordBoundaryNext wbnext, void *wbstate);
size_t unicode_strupper(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strfold(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
#endif /* UNICODE_CASE_H */
--
2.43.0
v5-0004-Validating-iterator-friendly-UTF8-encoder-decoder.patch (text/x-patch, 5.2K) download
From 4f86b3ccfac7a55b905b3f5d8201c953a3a295f1 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v5 4/5] Validating, iterator-friendly UTF8 encoder/decoder
API.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/include/mb/pg_wchar.h | 160 +++++++++++++++++++++++++++++++++++++-
1 file changed, 158 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..d8ea77c3fe0 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,161 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if (nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+
+ if ((*src & 0xe0) == 0xc0)
+ nbytes = 2;
+ else if ((*src & 0xf0) == 0xe0)
+ nbytes = 3;
+ else if ((*src & 0xf8) == 0xf0)
+ nbytes = 4;
+ else
+ goto invalid;
+
+ /* truncated */
+ if (srclen < nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ if (nbytes == 2)
+ {
+ /* check continuation byte */
+ if ((src[1] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x1f) << 6) |
+ (src[1] & 0x3f));
+
+ /* overlong */
+ if (codepoint < 0x0080)
+ goto invalid;
+ }
+ else if (nbytes == 3)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x0f) << 12) |
+ ((src[1] & 0x3f) << 6) |
+ (src[2] & 0x3f));
+
+ /* overlong or surrogate half */
+ if (codepoint < 0x0800 ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+ }
+ else if (nbytes == 4)
+ {
+ /* check continuation bytes */
+ if ((src[1] & 0xc0) != 0x80 || (src[2] & 0xc0) != 0x80 ||
+ (src[3] & 0xc0) != 0x80)
+ goto invalid;
+
+ codepoint = (char32_t) (((src[0] & 0x07) << 18) |
+ ((src[1] & 0x3f) << 12) |
+ ((src[2] & 0x3f) << 6) |
+ (src[3] & 0x3f));
+
+ /* overlong or out-of-range */
+ if (codepoint < 0x10000 || codepoint > 0x10FFFF)
+ goto invalid;
+ }
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +571,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
v5-0005-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch (text/x-patch, 8.5K) download
From 25562a48cfbd93c25f23e2734d27307d6f5459d6 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v5 5/5] unicode_case.c: use new utf8encode/utf8decode APIs.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 10 +--
src/common/unicode/case_test.c | 45 +++++++++----
src/common/unicode_case.c | 77 +++++++++++------------
3 files changed, 77 insertions(+), 55 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..63516e7174f 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,22 +62,22 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
size_t prev_offset = wbstate->offset;
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
/* invalid UTF8 */
- if (wbstate->offset + ulen > wbstate->len)
+ if (ulen <= 0)
{
wbstate->init = true;
wbstate->offset = wbstate->len;
return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 08421d9e5ca..9461b56742b 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -52,24 +52,35 @@ initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
- while (wbstate->offset < wbstate->len &&
- wbstate->str[wbstate->offset] != '\0')
+ while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ int ulen;
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
+ /* invalid UTF8 */
+ if (ulen <= 0)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
@@ -179,7 +190,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
@@ -191,7 +202,7 @@ test_icu(void)
}
icu_test_simple(code);
- unicode_to_utf8(code, (unsigned char *) code_str);
+ utf8encode((unsigned char *) code_str, 5, code);
icu_test_full(code_str);
successful++;
@@ -337,6 +348,18 @@ test_convert_case(void)
/* invalid UTF8: leading byte invalid length */
needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: surrogates */
+ needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: continuation with no leading byte */
+ needed = unicode_strfold(NULL, 0, "abc\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: out of range */
+ needed = unicode_strfold(NULL, 0, "abc\xF5\x80\x80\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: overlong */
+ needed = unicode_strfold(NULL, 0, "abc\xC1\xBFxyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 24753aaab09..4d8ee71e8dc 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -304,10 +298,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -352,13 +354,10 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
/* now at leading byte of previous sequence */
Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
@@ -367,18 +366,18 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
}
- ulen = utf8_mblen((const unsigned char *) str + offset);
+ ulen = utf8decode(&curr, (const unsigned char *) str + offset,
+ len - offset);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
/* iterate forward looking for following character */
for (int i = offset + ulen; i < len;)
{
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-24 08:44 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
1 sibling, 1 reply; 27+ messages in thread
From: Chao Li @ 2026-06-24 08:44 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Jun 24, 2026, at 13:45, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Mon, 2026-06-22 at 19:02 -0700, Jeff Davis wrote:
>> v4 attached.
>
> v5 attached.
>
> There's an extra patch 0002 to fix a logic bug when handling final
> sigma (only affects the builtin pg_unicode_fast locale), which I think
> should be backported to 18.
>
> Also added tests.
>
> Regards,
> Jeff Davis
>
> <v5-0001-unicode_case.c-defend-against-invalid-UTF8.patch><v5-0002-pg_unicode_fast-fix-final-sigma-logic.patch><v5-0003-unicode_case.c-change-API-to-signal-UTF8-decoding.patch><v5-0004-Validating-iterator-friendly-UTF8-encoder-decoder.patch><v5-0005-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch>
There is a compile warning against pg_wchar.h in 0004:
```
../../../src/include/mb/pg_wchar.h:523:11: warning: variable 'codepoint' is used uninitialized whenever 'if' condition is false [-Wsometimes-uninitialized]
523 | else if (nbytes == 4)
| ^~~~~~~~~~~
../../../src/include/mb/pg_wchar.h:540:16: note: uninitialized use occurs here
540 | *pcodepoint = codepoint;
| ^~~~~~~~~
../../../src/include/mb/pg_wchar.h:523:7: note: remove the 'if' if its condition is always true
523 | else if (nbytes == 4)
| ^~~~~~~~~~~~~~~~
524 | {
../../../src/include/mb/pg_wchar.h:469:20: note: initialize the variable 'codepoint' to silence this warning
469 | char32_t codepoint;
| ^
| = 0
1 warning generated.
```
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-24 09:29 Ayush Tiwari <ayushtiwari.slg01@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
1 sibling, 1 reply; 27+ messages in thread
From: Ayush Tiwari @ 2026-06-24 09:29 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
Hi,
On Wed, 24 Jun 2026 at 11:15, Jeff Davis <pgsql@j-davis.com> wrote:
> On Mon, 2026-06-22 at 19:02 -0700, Jeff Davis wrote:
> > v4 attached.
>
> v5 attached.
>
> There's an extra patch 0002 to fix a logic bug when handling final
> sigma (only affects the builtin pg_unicode_fast locale), which I think
> should be backported to 18.
>
> Also added tests.
>
Thanks for the patch!
I took a look at the v5 series and tried it locally. The split between the
backpatchable defensive change, the final-sigma fix, and the newer
utf8encode/utf8decode API work for master makes sense to me.
On my machine, case-check fails before reaching the new invalid-UTF8
assertions. I think it's because PostgreSQL's Unicode tables are 17.0 while
my
system ICU is 15.1, so the exhaustive ICU comparison in test_icu() hits a
changed mapping and exits first.
Would it be worth running test_convert_case() before
test_icu()? It wouldn't make case-check pass on a mismatched-ICU system,
but it
would at least let the non-ICU conversion and the new invalid-UTF8 cases run
before the ICU comparison aborts.
Regards,
Ayush
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-24 21:57 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-06-24 21:57 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Wed, 2026-06-24 at 16:44 +0800, Chao Li wrote:
> There is a compile warning against pg_wchar.h in 0004:
Fixed. I also used a loop in utf8decode() which is slightly smaller,
which is good if we intend it to be inlined by a lot of callers.
Regards,
Jeff Davis
Attachments: download all (5 files, .tar.gz)
v5-0001-unicode_case.c-defend-against-invalid-UTF8.patch (text/x-patch, 5.5K) download
From 9a9bfbc4f3866d77e48933df052086a2558e7263 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:30:01 -0700
Subject: [PATCH v5 1/5] unicode_case.c: defend against invalid UTF8.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 17
---
src/backend/utils/adt/pg_locale_builtin.c | 24 ++++++++---
src/common/unicode/case_test.c | 8 ++++
src/common/unicode_case.c | 52 +++++++++++++++++++----
3 files changed, 70 insertions(+), 14 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 01d4f55b07e..7c36fd5091b 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,21 +62,33 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ /* invalid UTF8 */
+ if (wbstate->offset + ulen > wbstate->len)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+ u = utf8_to_unicode((const unsigned char *) wbstate->str +
+ wbstate->offset);
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index a0dbf00b671..31ea94513bf 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -296,6 +296,8 @@ tfunc_fold(char *dst, size_t dstsize, const char *src,
static void
test_convert_case(void)
{
+ size_t needed;
+
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
/* test adjust-to-cased behavior */
@@ -320,6 +322,12 @@ test_convert_case(void)
/* U+FF11 FULLWIDTH ONE is alphanumeric for full case mapping */
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
+ /* invalid UTF8: truncated multibyte sequence */
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
+ Assert(needed == 3);
+ /* invalid UTF8: invalid byte */
+ needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
+ Assert(needed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index d6ee00b7d9c..42eb7d22211 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -189,6 +189,22 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL);
}
+/* local version of pg_utf_mblen() to be inlinable */
+static int
+utf8_mblen(const unsigned char *s)
+{
+ if ((*s & 0x80) == 0)
+ return 1;
+ else if ((*s & 0xe0) == 0xc0)
+ return 2;
+ else if ((*s & 0xf0) == 0xe0)
+ return 3;
+ else if ((*s & 0xf8) == 0xf0)
+ return 4;
+ else
+ return -1;
+}
+
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -227,12 +243,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- char32_t u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
- int u1len = unicode_utf8len(u1);
+ int u1len = utf8_mblen((const unsigned char *) src + srcoff);
+ char32_t u1;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ /* invalid UTF8 */
+ if (u1len < 0 || srcoff + u1len > srclen)
+ break;
+
+ u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
+
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -316,7 +338,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -327,8 +356,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
/* end of string is not followed by a Cased character */
@@ -340,7 +369,14 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
{
- char32_t curr = utf8_to_unicode(str + i);
+ int u1len = utf8_mblen((const unsigned char *) str + i);
+ char32_t curr;
+
+ /* invalid UTF8 */
+ if (u1len < 0 || i + u1len > len)
+ return false;
+
+ curr = utf8_to_unicode(str + i);
if (pg_u_prop_case_ignorable(curr))
continue;
@@ -351,8 +387,8 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
else if ((str[i] & 0xC0) == 0x80)
continue;
-
- Assert(false); /* invalid UTF-8 */
+ else
+ return false; /* invalid UTF8 */
}
return true;
--
2.43.0
v5-0002-pg_unicode_fast-fix-final-sigma-logic.patch (text/x-patch, 5.3K) download
From c389e8f3d47c51183702f468846c0ad8c33beaae Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Tue, 23 Jun 2026 17:09:49 -0700
Subject: [PATCH v5 2/5] pg_unicode_fast: fix final sigma logic.
If the string is preceded only by Case Ignorable characters, don't
consider it to be a final sigma.
In the process, refactor so that the preceding and following
characters are found first, and then the rule is applied, to improve
clarity.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
Backpatch-through: 18
---
src/common/unicode_case.c | 88 ++++++++++------------
src/test/regress/expected/collate.utf8.out | 6 ++
src/test/regress/sql/collate.utf8.sql | 1 +
3 files changed, 47 insertions(+), 48 deletions(-)
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 42eb7d22211..dd5b3ba86d0 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -323,75 +323,67 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
* 3-17. The character at the given offset must be directly preceded by a
* Cased character, and must not be directly followed by a Cased character.
*
- * Case_Ignorable characters are ignored. NB: some characters may be both
+ * Case_Ignorable characters are ignored. Neither beginning of string nor end
+ * of string are considered Cased characters. NB: some characters may be both
* Cased and Case_Ignorable, in which case they are ignored.
*/
static bool
check_final_sigma(const unsigned char *str, size_t len, size_t offset)
{
- /* the start of the string is not preceded by a Cased character */
- if (offset == 0)
- return false;
+ bool preceded_by_cased = false;
+ bool followed_by_cased = false;
+ char32_t curr;
+ int ulen;
- /* iterate backwards, looking for Cased character */
- for (int i = offset - 1; i >= 0; i--)
+ /* iterate backwards looking for preceding character */
+ for (int i = offset; i > 0;)
{
- if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
- {
- int u1len = utf8_mblen((const unsigned char *) str + i);
- char32_t curr;
+ /* skip backwards through continuation bytes */
+ i--;
+ if ((str[i] & 0xC0) == 0x80)
+ continue;
- /* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
- return false;
+ /* now at leading byte of previous sequence */
+ Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- curr = utf8_to_unicode(str + i);
+ ulen = utf8_mblen((const unsigned char *) str + i);
- if (pg_u_prop_case_ignorable(curr))
- continue;
- else if (pg_u_prop_cased(curr))
- break;
- else
- return false;
+ /* invalid UTF8 */
+ if (ulen < 0 || i + ulen > len)
+ return false;
+
+ curr = utf8_to_unicode((const unsigned char *) str + i);
+
+ if (!pg_u_prop_case_ignorable(curr))
+ {
+ preceded_by_cased = pg_u_prop_cased(curr);
+ break;
}
- else if ((str[i] & 0xC0) == 0x80)
- continue;
- else
- return false; /* invalid UTF8 */
}
- /* end of string is not followed by a Cased character */
- if (offset == len)
- return true;
+ ulen = utf8_mblen((const unsigned char *) str + offset);
- /* iterate forwards, looking for Cased character */
- for (int i = offset + 1; i < len && str[i] != '\0'; i++)
+ /* iterate forward looking for following character */
+ for (int i = offset + ulen; i < len;)
{
- if ((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0)
- {
- int u1len = utf8_mblen((const unsigned char *) str + i);
- char32_t curr;
+ ulen = utf8_mblen((const unsigned char *) str + i);
- /* invalid UTF8 */
- if (u1len < 0 || i + u1len > len)
- return false;
+ /* invalid UTF8 */
+ if (ulen < 0 || i + ulen > len)
+ return false;
- curr = utf8_to_unicode(str + i);
+ curr = utf8_to_unicode((const unsigned char *) str + i);
- if (pg_u_prop_case_ignorable(curr))
- continue;
- else if (pg_u_prop_cased(curr))
- return false;
- else
- break;
+ if (!pg_u_prop_case_ignorable(curr))
+ {
+ followed_by_cased = pg_u_prop_cased(curr);
+ break;
}
- else if ((str[i] & 0xC0) == 0x80)
- continue;
- else
- return false; /* invalid UTF8 */
+
+ i += ulen;
}
- return true;
+ return (preceded_by_cased && !followed_by_cased);
}
/*
diff --git a/src/test/regress/expected/collate.utf8.out b/src/test/regress/expected/collate.utf8.out
index 0c3ab5c89b2..99fdc111fa4 100644
--- a/src/test/regress/expected/collate.utf8.out
+++ b/src/test/regress/expected/collate.utf8.out
@@ -263,6 +263,12 @@ SELECT lower('ᾼΣͅΑ' COLLATE PG_UNICODE_FAST); -- 0391 0345 03A3 0345 0391
ᾳσͅα
(1 row)
+SELECT lower(U&'\0300\03A3' COLLATE PG_UNICODE_FAST);
+ lower
+-------
+ ̀σ
+(1 row)
+
-- properties
SELECT 'xyz' ~ '[[:alnum:]]' COLLATE PG_UNICODE_FAST;
?column?
diff --git a/src/test/regress/sql/collate.utf8.sql b/src/test/regress/sql/collate.utf8.sql
index d6d14220ab3..22aecee3a60 100644
--- a/src/test/regress/sql/collate.utf8.sql
+++ b/src/test/regress/sql/collate.utf8.sql
@@ -128,6 +128,7 @@ SELECT lower('0Σ' COLLATE PG_UNICODE_FAST); -- 0030 03A3
SELECT lower('ΑΣΑ' COLLATE PG_UNICODE_FAST); -- 0391 03A3 0391
SELECT lower('ἈΣ̓Α' COLLATE PG_UNICODE_FAST); -- 0391 0343 03A3 0343 0391
SELECT lower('ᾼΣͅΑ' COLLATE PG_UNICODE_FAST); -- 0391 0345 03A3 0345 0391
+SELECT lower(U&'\0300\03A3' COLLATE PG_UNICODE_FAST);
-- properties
--
2.43.0
v5-0003-unicode_case.c-change-API-to-signal-UTF8-decoding.patch (text/x-patch, 11.8K) download
From 7f2494e90f041cb6e118c0a6a062b9e5364bc799 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 22 Jun 2026 16:31:38 -0700
Subject: [PATCH v5 3/5] unicode_case.c: change API to signal UTF8 decoding
error.
Errors at this point are not expected, but if encountered, signal to
the caller so it can raise the appropriate error.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 50 +++++++++++++++++++----
src/common/unicode/case_test.c | 37 ++++++++++-------
src/common/unicode_case.c | 46 ++++++++++++---------
src/include/common/unicode_case.h | 8 ++--
4 files changed, 94 insertions(+), 47 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 7c36fd5091b..5619daf43c3 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -98,8 +98,16 @@ static size_t
strlower_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strlower(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strlower(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
@@ -114,26 +122,50 @@ strtitle_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
.init = false,
.prev_alnum = false,
};
+ size_t consumed;
+ size_t result;
- return unicode_strtitle(dest, destsize, src, srclen,
- locale->builtin.casemap_full,
- initcap_wbnext, &wbstate);
+ result = unicode_strtitle(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full,
+ initcap_wbnext, &wbstate);
+
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strupper_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strupper(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strupper(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static size_t
strfold_builtin(char *dest, size_t destsize, const char *src, size_t srclen,
pg_locale_t locale)
{
- return unicode_strfold(dest, destsize, src, srclen,
- locale->builtin.casemap_full);
+ size_t consumed;
+ size_t result;
+
+ result = unicode_strfold(dest, destsize, src, srclen, &consumed,
+ locale->builtin.casemap_full);
+ if (consumed < srclen)
+ report_invalid_encoding(GetDatabaseEncoding(), src + consumed,
+ srclen - consumed);
+
+ return result;
}
static bool
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 31ea94513bf..08421d9e5ca 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -115,6 +115,7 @@ icu_test_full(char *str)
char icu_fold[BUFSZ];
UErrorCode status;
size_t len = strlen(str);
+ size_t consumed;
/* full case mapping doesn't use posix semantics */
struct WordBoundaryState wbstate = {
@@ -126,10 +127,10 @@ icu_test_full(char *str)
.prev_alnum = false,
};
- unicode_strlower(lower, BUFSZ, str, len, true);
- unicode_strtitle(title, BUFSZ, str, len, true, initcap_wbnext, &wbstate);
- unicode_strupper(upper, BUFSZ, str, len, true);
- unicode_strfold(fold, BUFSZ, str, len, true);
+ unicode_strlower(lower, BUFSZ, str, len, &consumed, true);
+ unicode_strtitle(title, BUFSZ, str, len, &consumed, true, initcap_wbnext, &wbstate);
+ unicode_strupper(upper, BUFSZ, str, len, &consumed, true);
+ unicode_strfold(fold, BUFSZ, str, len, &consumed, true);
status = U_ZERO_ERROR;
ucasemap_utf8ToLower(casemap, icu_lower, BUFSZ, str, len, &status);
status = U_ZERO_ERROR;
@@ -260,13 +261,16 @@ static size_t
tfunc_lower(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strlower(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strlower(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_title(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
+ size_t consumed;
struct WordBoundaryState wbstate = {
.str = src,
.len = srclen,
@@ -275,28 +279,33 @@ tfunc_title(char *dst, size_t dstsize, const char *src,
.prev_alnum = false,
};
- return unicode_strtitle(dst, dstsize, src, srclen, true, initcap_wbnext,
- &wbstate);
+ return unicode_strtitle(dst, dstsize, src, srclen, &consumed, true,
+ initcap_wbnext, &wbstate);
}
static size_t
tfunc_upper(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strupper(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strupper(dst, dstsize, src, srclen, &consumed, true);
}
static size_t
tfunc_fold(char *dst, size_t dstsize, const char *src,
size_t srclen)
{
- return unicode_strfold(dst, dstsize, src, srclen, true);
+ size_t consumed;
+
+ return unicode_strfold(dst, dstsize, src, srclen, &consumed, true);
}
static void
test_convert_case(void)
{
size_t needed;
+ size_t consumed;
/* test string with no case changes */
test_convert(tfunc_lower, "√∞", "√∞");
@@ -323,11 +332,11 @@ test_convert_case(void)
test_convert(tfunc_title, "\uFF11a", "\uFF11a");
/* invalid UTF8: truncated multibyte sequence */
- needed = unicode_strfold(NULL, 0, "abc\xCE", 4, false);
- Assert(needed == 3);
- /* invalid UTF8: invalid byte */
- needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, false);
- Assert(needed == 3);
+ needed = unicode_strfold(NULL, 0, "abc\xCE", 4, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: leading byte invalid length */
+ needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index dd5b3ba86d0..24753aaab09 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -40,8 +40,8 @@ static const char32_t *const casekind_map[NCaseKind] =
static char32_t find_case_map(char32_t ucs, const char32_t *map);
static size_t convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate);
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate);
static enum CaseMapResult casemap(char32_t u1, CaseKind casekind, bool full,
const char *src, size_t srclen, size_t srcoff,
char32_t *simple, const char32_t **special);
@@ -82,7 +82,8 @@ unicode_casefold_simple(char32_t code)
* unicode_strlower()
*
* Convert src to lowercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -98,17 +99,18 @@ unicode_casefold_simple(char32_t code)
*/
size_t
unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseLower, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseLower, full,
+ NULL, NULL);
}
/*
* unicode_strtitle()
*
* Convert src to titlecase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -134,17 +136,19 @@ unicode_strlower(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full, WordBoundaryNext wbnext, void *wbstate)
+ size_t *pconsumed, bool full, WordBoundaryNext wbnext,
+ void *wbstate)
{
- return convert_case(dst, dstsize, src, srclen, CaseTitle, full, wbnext,
- wbstate);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseTitle, full,
+ wbnext, wbstate);
}
/*
* unicode_strupper()
*
* Convert src to uppercase, and return the result length (not including
- * terminating NUL).
+ * terminating NUL). Sets *pconsumed to the amount of src successfully
+ * consumed; if less than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -160,17 +164,18 @@ unicode_strtitle(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseUpper, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseUpper, full,
+ NULL, NULL);
}
/*
* unicode_strfold()
*
* Case fold src, and return the result length (not including terminating
- * NUL).
+ * NUL). Sets *pconsumed to the amount of src successfully consumed; if less
+ * than srclen, indicates a decoding error.
*
* String src must be encoded in UTF-8.
*
@@ -183,10 +188,10 @@ unicode_strupper(char *dst, size_t dstsize, const char *src, size_t srclen,
*/
size_t
unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
- bool full)
+ size_t *pconsumed, bool full)
{
- return convert_case(dst, dstsize, src, srclen, CaseFold, full, NULL,
- NULL);
+ return convert_case(dst, dstsize, src, srclen, pconsumed, CaseFold, full,
+ NULL, NULL);
}
/* local version of pg_utf_mblen() to be inlinable */
@@ -223,8 +228,8 @@ utf8_mblen(const unsigned char *s)
*/
static size_t
convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
- CaseKind str_casekind, bool full, WordBoundaryNext wbnext,
- void *wbstate)
+ size_t *pconsumed, CaseKind str_casekind, bool full,
+ WordBoundaryNext wbnext, void *wbstate)
{
/* character CaseKind varies while titlecasing */
CaseKind chr_casekind = str_casekind;
@@ -315,6 +320,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
if (result_len < dstsize)
dst[result_len] = '\0';
+ *pconsumed = srcoff;
return result_len;
}
diff --git a/src/include/common/unicode_case.h b/src/include/common/unicode_case.h
index 03add78cabe..1cbc0c14bc2 100644
--- a/src/include/common/unicode_case.h
+++ b/src/include/common/unicode_case.h
@@ -21,13 +21,13 @@ char32_t unicode_titlecase_simple(char32_t code);
char32_t unicode_uppercase_simple(char32_t code);
char32_t unicode_casefold_simple(char32_t code);
size_t unicode_strlower(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strtitle(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full,
+ size_t srclen, size_t *pconsumed, bool full,
WordBoundaryNext wbnext, void *wbstate);
size_t unicode_strupper(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
size_t unicode_strfold(char *dst, size_t dstsize, const char *src,
- size_t srclen, bool full);
+ size_t srclen, size_t *pconsumed, bool full);
#endif /* UNICODE_CASE_H */
--
2.43.0
v5-0004-Validating-iterator-friendly-UTF8-encoder-decoder.patch (text/x-patch, 4.8K) download
From 08acf6743051710355b6c8c6c9102a7922a3d163 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v5 4/5] Validating, iterator-friendly UTF8 encoder/decoder
API.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/include/mb/pg_wchar.h | 139 +++++++++++++++++++++++++++++++++++++-
1 file changed, 137 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..bc445be0678 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,140 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if ((size_t) nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+ char32_t min;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+ else if ((*src & 0xe0) == 0xc0)
+ {
+ nbytes = 2;
+ min = 0x80;
+ codepoint = (char32_t) src[0] & 0x1f;
+ }
+ else if ((*src & 0xf0) == 0xe0)
+ {
+ nbytes = 3;
+ min = 0x800;
+ codepoint = (char32_t) src[0] & 0x0f;
+ }
+ else if ((*src & 0xf8) == 0xf0)
+ {
+ nbytes = 4;
+ min = 0x10000;
+ codepoint = (char32_t) src[0] & 0x07;
+ }
+ else
+ goto invalid; /* invalid leading byte */
+
+ /* truncated multibyte sequence */
+ if (srclen < (size_t) nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ for (int i = 1; i < nbytes; i++)
+ {
+ if ((src[i] & 0xc0) != 0x80)
+ goto invalid;
+ codepoint = (codepoint << 6) | (src[i] & 0x3f);
+ }
+
+ /* reject overlong, surrogate, and out-of-range */
+ if (codepoint < min || codepoint > 0x10FFFF ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +550,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
v5-0005-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch (text/x-patch, 8.5K) download
From 91a0f3cac34d3a4030cd8adf8553261f33804d6d Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v5 5/5] unicode_case.c: use new utf8encode/utf8decode APIs.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 10 +--
src/common/unicode/case_test.c | 45 +++++++++----
src/common/unicode_case.c | 77 +++++++++++------------
3 files changed, 77 insertions(+), 55 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..63516e7174f 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,22 +62,22 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
size_t prev_offset = wbstate->offset;
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
/* invalid UTF8 */
- if (wbstate->offset + ulen > wbstate->len)
+ if (ulen <= 0)
{
wbstate->init = true;
wbstate->offset = wbstate->len;
return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 08421d9e5ca..9461b56742b 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -52,24 +52,35 @@ initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
- while (wbstate->offset < wbstate->len &&
- wbstate->str[wbstate->offset] != '\0')
+ while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ int ulen;
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
+ /* invalid UTF8 */
+ if (ulen <= 0)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
@@ -179,7 +190,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
@@ -191,7 +202,7 @@ test_icu(void)
}
icu_test_simple(code);
- unicode_to_utf8(code, (unsigned char *) code_str);
+ utf8encode((unsigned char *) code_str, 5, code);
icu_test_full(code_str);
successful++;
@@ -337,6 +348,18 @@ test_convert_case(void)
/* invalid UTF8: leading byte invalid length */
needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: surrogates */
+ needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: continuation with no leading byte */
+ needed = unicode_strfold(NULL, 0, "abc\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: out of range */
+ needed = unicode_strfold(NULL, 0, "abc\xF5\x80\x80\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: overlong */
+ needed = unicode_strfold(NULL, 0, "abc\xC1\xBFxyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 24753aaab09..4d8ee71e8dc 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -304,10 +298,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -352,13 +354,10 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
/* now at leading byte of previous sequence */
Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
@@ -367,18 +366,18 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
}
- ulen = utf8_mblen((const unsigned char *) str + offset);
+ ulen = utf8decode(&curr, (const unsigned char *) str + offset,
+ len - offset);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
/* iterate forward looking for following character */
for (int i = offset + ulen; i < len;)
{
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-24 23:43 Jeff Davis <pgsql@j-davis.com>
parent: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
0 siblings, 0 replies; 27+ messages in thread
From: Jeff Davis @ 2026-06-24 23:43 UTC (permalink / raw)
To: Ayush Tiwari <ayushtiwari.slg01@gmail.com>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
On Wed, 2026-06-24 at 14:59 +0530, Ayush Tiwari wrote:
> I took a look at the v5 series and tried it locally. The split
> between the
> backpatchable defensive change, the final-sigma fix, and the newer
> utf8encode/utf8decode API work for master makes sense to me.
Thank you for taking a look.
> On my machine, case-check fails before reaching the new invalid-UTF8
> assertions. I think it's because PostgreSQL's Unicode tables are 17.0
> while my
> system ICU is 15.1, so the exhaustive ICU comparison in test_icu()
> hits a
> changed mapping and exits first.
Yes, that test is a bit awkward because most of it is a comparison to
results from ICU, so if ICU is unavailable or based on an older version
of Unicode, then the test doesn't work.
> Would it be worth running test_convert_case() before
> test_icu()?
If we want to make those independent of ICU, I think we'd move them to
a regular test suite that's exercised everywhere (not just as part of
the 'update-unicode' target). But if we did so, that would be a very
small test suite, because most of the results are already checked in
the normal SQL tests. What makes these tests different is that they are
exercising invalid UTF8 behavior, which we don't expect to happen
through ordinary SQL.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-25 04:10 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Chao Li @ 2026-06-25 04:10 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Jun 25, 2026, at 05:57, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Wed, 2026-06-24 at 16:44 +0800, Chao Li wrote:
>> There is a compile warning against pg_wchar.h in 0004:
>
> Fixed. I also used a loop in utf8decode() which is slightly smaller,
> which is good if we intend it to be inlined by a lot of callers.
>
> Regards,
> Jeff Davis
>
> <v5-0001-unicode_case.c-defend-against-invalid-UTF8.patch><v5-0002-pg_unicode_fast-fix-final-sigma-logic.patch><v5-0003-unicode_case.c-change-API-to-signal-UTF8-decoding.patch><v5-0004-Validating-iterator-friendly-UTF8-encoder-decoder.patch><v5-0005-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch>
I just reviewed v5-0001 and got one concern.
In initcap_wbnext(), the new check only verifies that the input has enough bytes:
```
if (wbstate->offset + ulen > wbstate->len)
```
What about an invalid continuation byte, for example "\xCE "? In this case, pg_utf_mblen() sees \xCE, so ulen will be 2. Since there is still one more byte, the length check won't catch the invalid continuation byte \x20, and the code will proceed to utf8_to_unicode().
Looking at utf8_to_unicode():
```
static inline char32_t
utf8_to_unicode(const unsigned char *c)
{
if ((*c & 0x80) == 0)
return (char32_t) c[0];
else if ((*c & 0xe0) == 0xc0)
return (char32_t) (((c[0] & 0x1f) << 6) |
(c[1] & 0x3f));
else if ((*c & 0xf0) == 0xe0)
return (char32_t) (((c[0] & 0x0f) << 12) |
((c[1] & 0x3f) << 6) |
(c[2] & 0x3f));
else if ((*c & 0xf8) == 0xf0)
return (char32_t) (((c[0] & 0x07) << 18) |
((c[1] & 0x3f) << 12) |
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
return PG_INVALID_CODEPOINT;
}
```
For "\xCE ", it will take this branch:
```
else if ((*c & 0xe0) == 0xc0)
return (char32_t) (((c[0] & 0x1f) << 6) |
(c[1] & 0x3f));
```
This uses the second byte, \x20, without validating. So it looks like the patch prevents reading past the end of the string, but it may not fully defend against invalid UTF-8 sequences.
Am I missing anything?
(I will continue to review 0002 tomorrow.)
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-25 17:38 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-06-25 17:38 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Thu, 2026-06-25 at 12:10 +0800, Chao Li wrote:
> This uses the second byte, \x20, without validating. So it looks like
> the patch prevents reading past the end of the string, but it may not
> fully defend against invalid UTF-8 sequences.
Correct. We don't do full UTF8 validation until the last patch in the
series, which is not being backported.
Trying to do full validation in the backbranches seems more likely to
cause problems than prevent them. We aren't expecting invalid UTF8, but
in the event it got there somehow (perhaps from an old upgraded
instance), throwing errors after a minor release is probably not
helpful.
Even in master, I am not 100% sure we want to detect other kinds of
validation errors while processing the UTF8. By the time we are using
the value, maybe truncated multibyte sequences are the only thing we
care about, and we just need to be sure the code can handle anything
that fits in a char32_t.
Another thing to consider is an embedded NUL character, which is valid
UTF8 but not valid in a TEXT value.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-26 04:38 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 2 replies; 27+ messages in thread
From: Chao Li @ 2026-06-26 04:38 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Jun 26, 2026, at 01:38, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Thu, 2026-06-25 at 12:10 +0800, Chao Li wrote:
>> This uses the second byte, \x20, without validating. So it looks like
>> the patch prevents reading past the end of the string, but it may not
>> fully defend against invalid UTF-8 sequences.
>
> Correct. We don't do full UTF8 validation until the last patch in the
> series, which is not being backported.
>
Sounds like 0001 will be back patched. In that case, the commit message "defend against invalid UTF8” seems too broad. Does it make sense to add some brief description about the defend behavior to the function header comment and the commit message?
Then I continue to review 0002-0005:
0002 - overall looks good. A small comment is:
```
+ for (int i = offset; i > 0;)
+ for (int i = offset + ulen; i < len;)
```
As offset is of type size_t, the loop variable i is better to be size_t.
0003 - looks good.
0004 - looks good. This commit introduces a new helper utf8decode() that will resolve my previous concern on 0001.
0005 - Mostly looks good. This commit applies the new help and my previous concern is resolved. But from what you talked, I guess 0004 and 0005 will only be pushed to HEAD.
Just one tiny comment on 0005:
```
+ /* invalid UTF8: surrogates */
+ needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
```
This test passes a 10-char string but uses 7 as srclen. I know that doesn’t affect the test result, but it just adds unnecessary confusion to readers. So maybe change 7 to 10 to reflect to the real string length.
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-27 14:02 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
1 sibling, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-06-27 14:02 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Fri, 2026-06-26 at 12:38 +0800, Chao Li wrote:
> Sounds like 0001 will be back patched. In that case, the commit
> message "defend against invalid UTF8” seems too broad. Does it make
> sense to add some brief description about the defend behavior to the
> function header comment and the commit message?
Right. Would "defend against truncated byte sequences" or "defend
against truncated UTF8" be better wording?
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-06-28 22:29 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 0 replies; 27+ messages in thread
From: Chao Li @ 2026-06-28 22:29 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Jun 27, 2026, at 22:02, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Fri, 2026-06-26 at 12:38 +0800, Chao Li wrote:
>> Sounds like 0001 will be back patched. In that case, the commit
>> message "defend against invalid UTF8” seems too broad. Does it make
>> sense to add some brief description about the defend behavior to the
>> function header comment and the commit message?
>
> Right. Would "defend against truncated byte sequences" or "defend
> against truncated UTF8" be better wording?
>
> Regards,
> Jeff Davis
>
Yes, I think they are better. “Truncated” is more specific than “invalid”. I'm slightly more keen on the second phrase.
Best regards,
--
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-08 00:56 Jeff Davis <pgsql@j-davis.com>
parent: Chao Li <li.evan.chao@gmail.com>
1 sibling, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-07-08 00:56 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Fri, 2026-06-26 at 12:38 +0800, Chao Li wrote:
> Sounds like 0001 will be back patched. In that case, the commit
> message "defend against invalid UTF8” seems too broad. Does it make
> sense to add some brief description about the defend behavior to the
> function header comment and the commit message?
Committed 0001 and backported to 17. I think adding too much
explanation about behavior we don't expect to actually see would just
add confusion.
> Then I continue to review 0002-0005:
>
> 0002 - overall looks good. A small comment is:
> ```
> + for (int i = offset; i > 0;)
>
> + for (int i = offset + ulen; i < len;)
> ```
Committed 0002 and backported to 18. The 'int' is pre-existing, so I
left it as-is.
> 0003 - looks good.
Committed to master only.
> 0004 - looks good. This commit introduces a new helper utf8decode()
> that will resolve my previous concern on 0001.
>
> 0005 - Mostly looks good. This commit applies the new help and my
> previous concern is resolved. But from what you talked, I guess 0004
> and 0005 will only be pushed to HEAD.
>
> Just one tiny comment on 0005:
> ```
> + /* invalid UTF8: surrogates */
> + needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7,
> &consumed, false);
> + Assert(needed == 3 && consumed == 3);
> ```
>
> This test passes a 10-char string but uses 7 as srclen. I know that
> doesn’t affect the test result, but it just adds unnecessary
> confusion to readers. So maybe change 7 to 10 to reflect to the real
> string length.
Thank you, attached with fix.
I'd like to wait for more comments before I commit these last two
patches, to see if the functions are generally useful for other callers
as well.
Regards,
Jeff Davis
Attachments: download all (2 files, .tar.gz)
v6-0001-Validating-iterator-friendly-UTF8-encoder-decoder.patch (text/x-patch, 4.8K) download
From 5ab174963e2ab04455e59ea5b077238687151449 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 4 Jun 2026 12:08:51 -0700
Subject: [PATCH v6 1/2] Validating, iterator-friendly UTF8 encoder/decoder
API.
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/include/mb/pg_wchar.h | 139 +++++++++++++++++++++++++++++++++++++-
1 file changed, 137 insertions(+), 2 deletions(-)
diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h
index deee2a832c3..7afbf25fc18 100644
--- a/src/include/mb/pg_wchar.h
+++ b/src/include/mb/pg_wchar.h
@@ -27,6 +27,11 @@
*/
typedef unsigned int pg_wchar;
+/*
+ * Returned for decoding failures in utf8decode() and utf8_to_unicode().
+ */
+#define PG_INVALID_CODEPOINT 0xFFFFFFFF
+
/*
* Maximum byte length of multibyte characters in any backend encoding
*/
@@ -392,11 +397,140 @@ surrogate_pair_to_codepoint(char16_t first, char16_t second)
return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF);
}
+/*
+ * Encode the codepoint as UTF8 and return the number of bytes required. If
+ * the number of bytes required exceeds dstsize, just return the number of
+ * bytes required without modifying dst. If dstsize is zero, dst may be
+ * NULL. If codepoint is not a valid Unicode Scalar, return -1.
+ */
+static inline int
+utf8encode(unsigned char *dst, size_t dstsize, char32_t codepoint)
+{
+ int nbytes;
+
+ if (codepoint <= 0x7F)
+ nbytes = 1;
+ else if (codepoint <= 0x7FF)
+ nbytes = 2;
+ else if (codepoint <= 0xFFFF)
+ {
+ /* surrogate halves not valid for UTF8 */
+ if (codepoint >= 0xD800 && codepoint <= 0xDFFF)
+ return -1;
+ nbytes = 3;
+ }
+ else if (codepoint <= 0x10FFFF)
+ nbytes = 4;
+ else
+ return -1;
+
+ if ((size_t) nbytes > dstsize)
+ return nbytes;
+
+ if (codepoint <= 0x7F)
+ {
+ dst[0] = codepoint;
+ }
+ else if (codepoint <= 0x7FF)
+ {
+ dst[0] = 0xC0 | ((codepoint >> 6) & 0x1F);
+ dst[1] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0xFFFF)
+ {
+ dst[0] = 0xE0 | ((codepoint >> 12) & 0x0F);
+ dst[1] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[2] = 0x80 | (codepoint & 0x3F);
+ }
+ else if (codepoint <= 0x10FFFF)
+ {
+ dst[0] = 0xF0 | ((codepoint >> 18) & 0x07);
+ dst[1] = 0x80 | ((codepoint >> 12) & 0x3F);
+ dst[2] = 0x80 | ((codepoint >> 6) & 0x3F);
+ dst[3] = 0x80 | (codepoint & 0x3F);
+ }
+
+ return nbytes;
+}
+
+/*
+ * Decode the next Unicode codepoint from UTF8 at src, reading no more than
+ * srclen bytes (which must be at least 1). On success, *pcodepoint will be a
+ * valid Unicode Scalar; otherwise it will be set to PG_INVALID_CODEPOINT.
+ *
+ * Returns the number of bytes consumed. If srclen is not large enough
+ * (i.e. src is truncated in the middle of a sequence), returns 0. If invalid,
+ * returns -1.
+ */
+static inline int
+utf8decode(char32_t *pcodepoint, const unsigned char *src, size_t srclen)
+{
+ int nbytes;
+ char32_t codepoint;
+ char32_t min;
+
+ Assert(srclen >= 1);
+
+ if ((*src & 0x80) == 0)
+ {
+ *pcodepoint = (char32_t) src[0];
+ return 1;
+ }
+ else if ((*src & 0xe0) == 0xc0)
+ {
+ nbytes = 2;
+ min = 0x80;
+ codepoint = (char32_t) src[0] & 0x1f;
+ }
+ else if ((*src & 0xf0) == 0xe0)
+ {
+ nbytes = 3;
+ min = 0x800;
+ codepoint = (char32_t) src[0] & 0x0f;
+ }
+ else if ((*src & 0xf8) == 0xf0)
+ {
+ nbytes = 4;
+ min = 0x10000;
+ codepoint = (char32_t) src[0] & 0x07;
+ }
+ else
+ goto invalid; /* invalid leading byte */
+
+ /* truncated multibyte sequence */
+ if (srclen < (size_t) nbytes)
+ {
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return 0;
+ }
+
+ for (int i = 1; i < nbytes; i++)
+ {
+ if ((src[i] & 0xc0) != 0x80)
+ goto invalid;
+ codepoint = (codepoint << 6) | (src[i] & 0x3f);
+ }
+
+ /* reject overlong, surrogate, and out-of-range */
+ if (codepoint < min || codepoint > 0x10FFFF ||
+ (codepoint >= 0xD800 && codepoint <= 0xDFFF))
+ goto invalid;
+
+ *pcodepoint = codepoint;
+ return nbytes;
+
+invalid:
+ *pcodepoint = PG_INVALID_CODEPOINT;
+ return -1;
+}
+
/*
* Convert a UTF-8 character to a Unicode code point.
* This is a one-character version of pg_utf2wchar_with_len.
*
* No error checks here, c must point to a long-enough string.
+ *
+ * XXX: Callers should consider utf8decode() instead.
*/
static inline char32_t
utf8_to_unicode(const unsigned char *c)
@@ -416,13 +550,14 @@ utf8_to_unicode(const unsigned char *c)
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
- /* that is an invalid code on purpose */
- return 0xffffffff;
+ return PG_INVALID_CODEPOINT;
}
/*
* Map a Unicode code point to UTF-8. utf8string must have at least
* unicode_utf8len(c) bytes available.
+ *
+ * XXX: Callers should consider utf8encode() instead.
*/
static inline unsigned char *
unicode_to_utf8(char32_t c, unsigned char *utf8string)
--
2.43.0
v6-0002-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch (text/x-patch, 8.6K) download
From 2d9c03f786db067bfa0d89e8ab7f3b1ee5e1b0c0 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Fri, 19 Jun 2026 14:58:47 -0700
Subject: [PATCH v6 2/2] unicode_case.c: use new utf8encode/utf8decode APIs.
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/c355354e6c3f4a7aafb047361b73db247260fca0.camel@j-davis.com
---
src/backend/utils/adt/pg_locale_builtin.c | 10 +--
src/common/unicode/case_test.c | 45 +++++++++----
src/common/unicode_case.c | 77 +++++++++++------------
3 files changed, 77 insertions(+), 55 deletions(-)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..63516e7174f 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -62,22 +62,22 @@ initcap_wbnext(void *state)
while (wbstate->offset < wbstate->len)
{
- int ulen = pg_utf_mblen((const unsigned char *) wbstate->str +
- wbstate->offset);
+ int ulen;
char32_t u;
bool curr_alnum;
size_t prev_offset = wbstate->offset;
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
/* invalid UTF8 */
- if (wbstate->offset + ulen > wbstate->len)
+ if (ulen <= 0)
{
wbstate->init = true;
wbstate->offset = wbstate->len;
return prev_offset;
}
- u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
curr_alnum = pg_u_isalnum(u, wbstate->posix);
if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
diff --git a/src/common/unicode/case_test.c b/src/common/unicode/case_test.c
index 08421d9e5ca..22f3f9064b6 100644
--- a/src/common/unicode/case_test.c
+++ b/src/common/unicode/case_test.c
@@ -52,24 +52,35 @@ initcap_wbnext(void *state)
{
struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
- while (wbstate->offset < wbstate->len &&
- wbstate->str[wbstate->offset] != '\0')
+ while (wbstate->offset < wbstate->len)
{
- char32_t u = utf8_to_unicode((const unsigned char *) wbstate->str +
- wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ int ulen;
+ char32_t u;
+ bool curr_alnum;
+ size_t prev_offset = wbstate->offset;
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ ulen = utf8decode(&u, (const unsigned char *) wbstate->str + wbstate->offset,
+ wbstate->len - wbstate->offset);
+
+ /* invalid UTF8 */
+ if (ulen <= 0)
{
- size_t prev_offset = wbstate->offset;
+ wbstate->init = true;
+ wbstate->offset = wbstate->len;
+ return prev_offset;
+ }
+
+ curr_alnum = pg_u_isalnum(u, wbstate->posix);
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
wbstate->prev_alnum = curr_alnum;
return prev_offset;
}
- wbstate->offset += unicode_utf8len(u);
+ wbstate->offset += ulen;
}
return wbstate->len;
@@ -179,7 +190,7 @@ test_icu(void)
{
pg_unicode_category category = unicode_category(code);
- if (category != PG_U_UNASSIGNED)
+ if (category != PG_U_UNASSIGNED && category != PG_U_SURROGATE)
{
uint8_t icu_category = u_charType(code);
char code_str[5] = {0};
@@ -191,7 +202,7 @@ test_icu(void)
}
icu_test_simple(code);
- unicode_to_utf8(code, (unsigned char *) code_str);
+ utf8encode((unsigned char *) code_str, 5, code);
icu_test_full(code_str);
successful++;
@@ -337,6 +348,18 @@ test_convert_case(void)
/* invalid UTF8: leading byte invalid length */
needed = unicode_strfold(NULL, 0, "abc\xF8xyz", 7, &consumed, false);
Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: surrogates */
+ needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 9, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: continuation with no leading byte */
+ needed = unicode_strfold(NULL, 0, "abc\x80xyz", 7, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: out of range */
+ needed = unicode_strfold(NULL, 0, "abc\xF5\x80\x80\x80xyz", 10, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
+ /* invalid UTF8: overlong */
+ needed = unicode_strfold(NULL, 0, "abc\xC1\xBFxyz", 8, &consumed, false);
+ Assert(needed == 3 && consumed == 3);
#ifdef USE_ICU
icu_test_full("");
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 24753aaab09..4d8ee71e8dc 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -194,22 +194,6 @@ unicode_strfold(char *dst, size_t dstsize, const char *src, size_t srclen,
NULL, NULL);
}
-/* local version of pg_utf_mblen() to be inlinable */
-static int
-utf8_mblen(const unsigned char *s)
-{
- if ((*s & 0x80) == 0)
- return 1;
- else if ((*s & 0xe0) == 0xc0)
- return 2;
- else if ((*s & 0xf0) == 0xe0)
- return 3;
- else if ((*s & 0xf8) == 0xf0)
- return 4;
- else
- return -1;
-}
-
/*
* Implement Unicode Default Case Conversion algorithm.
*
@@ -248,18 +232,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
while (srcoff < srclen)
{
- int u1len = utf8_mblen((const unsigned char *) src + srcoff);
char32_t u1;
+ int u1len;
char32_t simple = 0;
const char32_t *special = NULL;
enum CaseMapResult casemap_result;
+ u1len = utf8decode(&u1, (const unsigned char *) src + srcoff,
+ srclen - srcoff);
+
/* invalid UTF8 */
- if (u1len < 0 || srcoff + u1len > srclen)
+ if (u1len <= 0)
break;
- u1 = utf8_to_unicode((const unsigned char *) src + srcoff);
-
if (str_casekind == CaseTitle)
{
if (srcoff == boundary)
@@ -280,6 +265,7 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
/* no mapping; copy bytes from src */
Assert(simple == 0);
Assert(special == NULL);
+
if (result_len + u1len <= dstsize)
memcpy(dst + result_len, src + srcoff, u1len);
@@ -289,11 +275,19 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
{
/* replace with single character */
char32_t u2 = simple;
- char32_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
Assert(special == NULL);
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -304,10 +298,18 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
{
char32_t u2 = special[i];
- size_t u2len = unicode_utf8len(u2);
+ int u2len;
+ size_t remaining = 0;
+ unsigned char *p = NULL;
+
+ if (dstsize > result_len)
+ {
+ remaining = dstsize - result_len;
+ p = (unsigned char *) dst + result_len;
+ }
- if (result_len + u2len <= dstsize)
- unicode_to_utf8(u2, (unsigned char *) dst + result_len);
+ u2len = utf8encode(p, remaining, u2);
+ Assert(u2len > 0);
result_len += u2len;
}
@@ -352,13 +354,10 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
/* now at leading byte of previous sequence */
Assert((str[i] & 0x80) == 0 || (str[i] & 0xC0) == 0xC0);
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
@@ -367,18 +366,18 @@ check_final_sigma(const unsigned char *str, size_t len, size_t offset)
}
}
- ulen = utf8_mblen((const unsigned char *) str + offset);
+ ulen = utf8decode(&curr, (const unsigned char *) str + offset,
+ len - offset);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
/* iterate forward looking for following character */
for (int i = offset + ulen; i < len;)
{
- ulen = utf8_mblen((const unsigned char *) str + i);
-
- /* invalid UTF8 */
- if (ulen < 0 || i + ulen > len)
- return false;
+ ulen = utf8decode(&curr, (const unsigned char *) str + i, len - i);
- curr = utf8_to_unicode((const unsigned char *) str + i);
+ if (ulen <= 0)
+ return false; /* invalid UTF8 */
if (!pg_u_prop_case_ignorable(curr))
{
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-08 16:48 Chao Li <li.evan.chao@gmail.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Chao Li @ 2026-07-08 16:48 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: pgsql-hackers
> On Jul 7, 2026, at 17:56, Jeff Davis <pgsql@j-davis.com> wrote:
>
> On Fri, 2026-06-26 at 12:38 +0800, Chao Li wrote:
>> Sounds like 0001 will be back patched. In that case, the commit
>> message "defend against invalid UTF8” seems too broad. Does it make
>> sense to add some brief description about the defend behavior to the
>> function header comment and the commit message?
>
> Committed 0001 and backported to 17. I think adding too much
> explanation about behavior we don't expect to actually see would just
> add confusion.
>
>> Then I continue to review 0002-0005:
>>
>> 0002 - overall looks good. A small comment is:
>> ```
>> + for (int i = offset; i > 0;)
>>
>> + for (int i = offset + ulen; i < len;)
>> ```
>
> Committed 0002 and backported to 18. The 'int' is pre-existing, so I
> left it as-is.
>
>> 0003 - looks good.
>
> Committed to master only.
>
>> 0004 - looks good. This commit introduces a new helper utf8decode()
>> that will resolve my previous concern on 0001.
>>
>> 0005 - Mostly looks good. This commit applies the new help and my
>> previous concern is resolved. But from what you talked, I guess 0004
>> and 0005 will only be pushed to HEAD.
>>
>> Just one tiny comment on 0005:
>> ```
>> + /* invalid UTF8: surrogates */
>> + needed = unicode_strfold(NULL, 0, "abc\xED\xA0\x81xyz", 7,
>> &consumed, false);
>> + Assert(needed == 3 && consumed == 3);
>> ```
>>
>> This test passes a 10-char string but uses 7 as srclen. I know that
>> doesn’t affect the test result, but it just adds unnecessary
>> confusion to readers. So maybe change 7 to 10 to reflect to the real
>> string length.
>
> Thank you, attached with fix.
>
> I'd like to wait for more comments before I commit these last two
> patches, to see if the functions are generally useful for other callers
> as well.
>
> Regards,
> Jeff Davis
>
> <v6-0001-Validating-iterator-friendly-UTF8-encoder-decoder.patch><v6-0002-unicode_case.c-use-new-utf8encode-utf8decode-APIs.patch>
V6 looks good to me.
Best regards,
—
Chao Li (Evan)
HighGo Software Co., Ltd.
https://www.highgo.com/
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-13 13:42 Tom Lane <tgl@sss.pgh.pa.us>
parent: Chao Li <li.evan.chao@gmail.com>
0 siblings, 1 reply; 27+ messages in thread
From: Tom Lane @ 2026-07-13 13:42 UTC (permalink / raw)
To: Chao Li <li.evan.chao@gmail.com>; +Cc: Jeff Davis <pgsql@j-davis.com>; pgsql-hackers
Chao Li <li.evan.chao@gmail.com> writes:
> On Jul 7, 2026, at 17:56, Jeff Davis <pgsql@j-davis.com> wrote:
>> I'd like to wait for more comments before I commit these last two
>> patches, to see if the functions are generally useful for other callers
>> as well.
> V6 looks good to me.
Coverity is unimpressed, although after reading 07211f64a I wonder if
this patch didn't simply trigger it to pick up a pre-existing problem
(it does seem to act that way sometimes). It now complains:
/srv/coverity/git/pgsql-git/postgresql/src/common/unicode_case.c: 324 in convert_case()
318 }
319
320 if (result_len < dstsize)
321 dst[result_len] = '\0';
322
323 *pconsumed = srcoff;
>>> CID 1696680: (INTEGER_OVERFLOW)
>>> "result_len", which might have overflowed, is returned from the function.
324 return result_len;
325 }
What I think this is pointing out is that even if the input string's
length fits in size_t, the length of the case-converted equivalent
string might not. The code won't write past dstsize, but it will
try to compute the full output length required, and it won't notice
if result_len overflows and wraps around.
I would have written this off as an unreachable edge case, but our
recent experience with unicode_normalize (cf. commit 066b7b144)
makes me hesitant to assume that case-folding can't result in
integer-multiple growth of the string length. If it can, the overflow
might be reachable on 32-bit platforms, resulting in a silently-broken
result (but no memory clobber AFAICS).
Catering for this case seems like it would require messy warts on the
casefolding functions' API. So if we can convince ourselves that
overflow can't really occur, I'd be content to add a comment
demonstrating that. But if we can't prove that, I think we need some
warts :-(
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-13 17:09 Jeff Davis <pgsql@j-davis.com>
parent: Tom Lane <tgl@sss.pgh.pa.us>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-07-13 17:09 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; Chao Li <li.evan.chao@gmail.com>; +Cc: pgsql-hackers
On Mon, 2026-07-13 at 09:42 -0400, Tom Lane wrote:
> What I think this is pointing out is that even if the input string's
> length fits in size_t, the length of the case-converted equivalent
> string might not. The code won't write past dstsize, but it will
> try to compute the full output length required, and it won't notice
> if result_len overflows and wraps around.
>
> I would have written this off as an unreachable edge case, but our
> recent experience with unicode_normalize (cf. commit 066b7b144)
> makes me hesitant to assume that case-folding can't result in
> integer-multiple growth of the string length. If it can, the
> overflow
> might be reachable on 32-bit platforms, resulting in a silently-
> broken
> result (but no memory clobber AFAICS).
From Unicode 17.0 section 5.18.2: "the maximum string expansion as a
result of case mapping in the Unicode Standard is three". Along with a
potential for 4X difference in byte length for a single code point in
UTF8, that makes a maximum of 12X expansion.
> Catering for this case seems like it would require messy warts on the
> casefolding functions' API. So if we can convince ourselves that
> overflow can't really occur, I'd be content to add a comment
> demonstrating that.
Based on the actual data I expect the worst case to be smaller,
probably 3X. That would be convenient because I think that means it
can't overflow: the largest text value we support is 1GB, while 3GB
would still fit in a size_t. Unfortunately, even if that's true today,
it would be hard to be confident it won't change in a future version of
Unicode.
> But if we can't prove that, I think we need some
> warts :-(
07211f64ac at least has an error path already, so we could set
*pconsumed=0 on overflow and return SIZE_MAX. It would be extra work
for the caller if it wants to distinguish between overflow and an
encoding error, but if we aren't expecting to hit it in practice, then
maybe it's OK to give an incorrect error.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-13 17:19 Tom Lane <tgl@sss.pgh.pa.us>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Tom Lane @ 2026-07-13 17:19 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
Jeff Davis <pgsql@j-davis.com> writes:
> On Mon, 2026-07-13 at 09:42 -0400, Tom Lane wrote:
>> What I think this is pointing out is that even if the input string's
>> length fits in size_t, the length of the case-converted equivalent
>> string might not.
> From Unicode 17.0 section 5.18.2: "the maximum string expansion as a
> result of case mapping in the Unicode Standard is three". Along with a
> potential for 4X difference in byte length for a single code point in
> UTF8, that makes a maximum of 12X expansion.
Well, there aren't going to be any ASCII characters that produce 12
bytes, and probably not any LATIN1 characters either, so I think we
could assume that the shortest input character that could produce
such a long output would be 3 bytes. So it might be safe to assume
that the expansion rate is not more than 4X, which would make this
okay (since as you say we can assume the input is < 1GB). Probably
needs a little bit of inspection of the LATIN1 cases though, ie
could any of those 2-byte characters produce more than 8 bytes?
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-13 17:20 Tom Lane <tgl@sss.pgh.pa.us>
parent: Tom Lane <tgl@sss.pgh.pa.us>
0 siblings, 1 reply; 27+ messages in thread
From: Tom Lane @ 2026-07-13 17:20 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
I wrote:
> Well, there aren't going to be any ASCII characters that produce 12
> bytes, and probably not any LATIN1 characters either, so I think we
> could assume that the shortest input character that could produce
> such a long output would be 3 bytes.
Ah sorry, I should have written BMP not LATIN1. Maybe it still holds
across that set, but I'm less sure.
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-14 03:26 Jeff Davis <pgsql@j-davis.com>
parent: Tom Lane <tgl@sss.pgh.pa.us>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-07-14 03:26 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
On Mon, 2026-07-13 at 13:20 -0400, Tom Lane wrote:
> I wrote:
> > Well, there aren't going to be any ASCII characters that produce 12
> > bytes, and probably not any LATIN1 characters either, so I think we
> > could assume that the shortest input character that could produce
> > such a long output would be 3 bytes.
>
> Ah sorry, I should have written BMP not LATIN1. Maybe it still holds
> across that set, but I'm less sure.
Attached a patch. I added some perl code to check that the expansion
factor of the UTF8 representation doesn't exceed 3, and added a static
assert in the C code to compare to MaxAllocSize.
Regards,
Jeff Davis
Attachments:
v1-0001-Clarify-use-of-size_t-for-case-mapping.patch (text/x-patch, 5.4K) download
From 1fa065c524600609a495433069f43bdbb88e2e05 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 13 Jul 2026 19:31:56 -0700
Subject: [PATCH v1] Clarify use of size_t for case mapping.
It cannot overflow, but add some static checking to be sure.
Reported-by: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://postgr.es/m/3213927.1783950167@sss.pgh.pa.us
---
src/backend/utils/adt/pg_locale_builtin.c | 10 ++++++
.../unicode/generate-unicode_case_table.pl | 32 +++++++++++++++++++
src/common/unicode_case.c | 6 ++++
src/include/common/unicode_case.h | 8 +++++
4 files changed, 56 insertions(+)
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..c2cf68a518b 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -17,9 +17,19 @@
#include "common/unicode_category.h"
#include "miscadmin.h"
#include "utils/builtins.h"
+#include "utils/memutils.h"
#include "utils/pg_locale.h"
#include "utils/syscache.h"
+/*
+ * The largest text value must fit in MaxAllocSize, but then may grow during
+ * case mapping. While the resulting string will not be representable as a new
+ * text value, we must at least be sure not to overflow a size_t while
+ * processing it.
+ */
+StaticAssertDecl(SIZE_MAX / MAX_CASE_UTF8_EXPANSION > MaxAllocSize,
+ "case mapping may overflow size_t");
+
extern pg_locale_t create_pg_locale_builtin(Oid collid,
MemoryContext context);
extern char *get_collation_actual_version_builtin(const char *collcollate);
diff --git a/src/common/unicode/generate-unicode_case_table.pl b/src/common/unicode/generate-unicode_case_table.pl
index c1b94f47f56..b53b84ebc88 100644
--- a/src/common/unicode/generate-unicode_case_table.pl
+++ b/src/common/unicode/generate-unicode_case_table.pl
@@ -11,6 +11,7 @@
use strict;
use warnings FATAL => 'all';
use Getopt::Long;
+use Encode qw(encode_utf8);
use FindBin;
use lib "$FindBin::RealBin/../../tools/";
@@ -25,6 +26,11 @@ my $output_table_file = "$output_path/unicode_case_table.h";
# of a single character. See Unicode section 5.18 "Case Mappings".
my $MAX_CASE_EXPANSION = 3;
+
+# The maximum expansion factor (in bytes) when case mapping a UTF8
+# string.
+my $MAX_CASE_UTF8_EXPANSION = 3;
+
my $FH;
my %simple = ();
@@ -44,6 +50,10 @@ while (my $line = <$FH>)
die "Simple_Titlecase $code out of range" if $simple_titlecase > 0x10FFFF;
die "Simple_Uppercase $code out of range" if $simple_uppercase > 0x10FFFF;
+ check_utf8_expansion($code, ($simple_uppercase));
+ check_utf8_expansion($code, ($simple_lowercase));
+ check_utf8_expansion($code, ($simple_titlecase));
+
if ($simple_lowercase || $simple_titlecase || $simple_uppercase)
{
$simple{$code} = {
@@ -113,6 +123,10 @@ while (my $line = <$FH>)
die "uppercase expansion for 0x$elts[0] exceeds maximum: '$elts[3]'"
if (scalar @upper) > $MAX_CASE_EXPANSION;
+ check_utf8_expansion($code, @lower);
+ check_utf8_expansion($code, @title);
+ check_utf8_expansion($code, @upper);
+
# pad arrays to a fixed length of 3
while (scalar @upper < $MAX_CASE_EXPANSION) { push @upper, 0x000000 }
while (scalar @lower < $MAX_CASE_EXPANSION) { push @lower, 0x000000 }
@@ -174,6 +188,8 @@ while (my $line = <$FH>)
die "unsupported status type '$status'"
if $status ne 'S' && $status ne 'C' && $status ne 'F';
+ check_utf8_expansion($code, @fold);
+
# initialize simple case mappings if they don't exist
$simple{$code} ||= {
Simple_Lowercase => $code,
@@ -522,6 +538,22 @@ EOS
close $OT;
+# compare byte length of UTF8 representation of the original codepoint
+# to the codepoint(s) it maps to, and track the largest expansion
+sub check_utf8_expansion
+{
+ my ($c, @a) = @_;
+
+ my $utf8len1 = length(encode_utf8(chr $c));
+ my $utf8len2 = length(encode_utf8(join '', map { chr } @a));
+ my $factor = 0;
+ $factor++ while ($factor * $utf8len1 < $utf8len2);
+
+ my $c_hex = sprintf "0x%06x", $c;
+ die "UTF8 expansion of $c_hex exceeds $MAX_CASE_UTF8_EXPANSION"
+ if ($factor > $MAX_CASE_UTF8_EXPANSION);
+}
+
# The function generates C code with a series of nested if-else conditions
# to search for the matching interval.
sub branch
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 399f08235a9..5f68154c8aa 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -237,6 +237,12 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
size_t result_len = 0;
size_t boundary = 0;
+ /*
+ * Must be guaranteed by caller to avoid overflow (text values limited to
+ * MaxAllocSize anyway).
+ */
+ Assert(srclen < SIZE_MAX / MAX_CASE_UTF8_EXPANSION);
+
Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
(str_casekind != CaseTitle && !wbnext && !wbstate));
diff --git a/src/include/common/unicode_case.h b/src/include/common/unicode_case.h
index 1cbc0c14bc2..a2b752667ca 100644
--- a/src/include/common/unicode_case.h
+++ b/src/include/common/unicode_case.h
@@ -14,6 +14,14 @@
#ifndef UNICODE_CASE_H
#define UNICODE_CASE_H
+/*
+ * The maximum expansion factor (in bytes) when case mapping a UTF8 string.
+ *
+ * Not guaranteed by the standard, but checked while constructing the mapping
+ * tables from Unicode data.
+ */
+#define MAX_CASE_UTF8_EXPANSION 3
+
typedef size_t (*WordBoundaryNext) (void *wbstate);
char32_t unicode_lowercase_simple(char32_t code);
--
2.43.0
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-15 21:39 Jeff Davis <pgsql@j-davis.com>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Jeff Davis @ 2026-07-15 21:39 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
On Mon, 2026-07-13 at 20:26 -0700, Jeff Davis wrote:
> Attached a patch. I added some perl code to check that the expansion
> factor of the UTF8 representation doesn't exceed 3, and added a
> static
> assert in the C code to compare to MaxAllocSize.
I plan to commit this soon. Only to master, unless you'd like me to
backport to silence some coverity warnings.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-15 22:25 Tom Lane <tgl@sss.pgh.pa.us>
parent: Jeff Davis <pgsql@j-davis.com>
0 siblings, 1 reply; 27+ messages in thread
From: Tom Lane @ 2026-07-15 22:25 UTC (permalink / raw)
To: Jeff Davis <pgsql@j-davis.com>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
Jeff Davis <pgsql@j-davis.com> writes:
> On Mon, 2026-07-13 at 20:26 -0700, Jeff Davis wrote:
>> Attached a patch. I added some perl code to check that the expansion
>> factor of the UTF8 representation doesn't exceed 3, and added a
>> static assert in the C code to compare to MaxAllocSize.
> I plan to commit this soon. Only to master, unless you'd like me to
> backport to silence some coverity warnings.
Master seems sufficient.
regards, tom lane
^ permalink raw reply [nested|flat] 27+ messages in thread
* Re: Small patch to improve safety of utf8_to_unicode().
@ 2026-07-16 21:47 Jeff Davis <pgsql@j-davis.com>
parent: Tom Lane <tgl@sss.pgh.pa.us>
0 siblings, 0 replies; 27+ messages in thread
From: Jeff Davis @ 2026-07-16 21:47 UTC (permalink / raw)
To: Tom Lane <tgl@sss.pgh.pa.us>; +Cc: Chao Li <li.evan.chao@gmail.com>; pgsql-hackers
On Wed, 2026-07-15 at 18:25 -0400, Tom Lane wrote:
> Master seems sufficient.
In that case I did a bit of refactoring to clean it up and put it in a
new unicode_limits.h header.
Regards,
Jeff Davis
Attachments:
v2-0001-Generate-unicode_limits.h.patch (text/x-patch, 17.9K) download
From b09c5bc0a437fe196940d7633fac4985e5538a06 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Mon, 13 Jul 2026 19:31:56 -0700
Subject: [PATCH v2] Generate unicode_limits.h.
To ensure we do not overflow a size_t while case mapping on a 32-bit
platform, we need to know the maximum amount a UTF8 string can expand.
Calculate that maximum while generating unicode tables as a part of
the update-unicode target, and output into a new header
unicode_limits.h. Minor refactoring along the way.
Add a StaticAssertDecl to check that a MaxAllocSize text value
expanded by that amount would still have a length that fits in
size_t. (We couldn't actually create a new text value out of that, but
we still need to avoid overflow.)
Reported-by: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://postgr.es/m/3213927.1783950167@sss.pgh.pa.us
---
src/backend/utils/adt/pg_locale_builtin.c | 11 +
src/common/unicode/Makefile | 6 +-
.../unicode/generate-unicode_case_table.pl | 194 ++++++++++++------
src/common/unicode/meson.build | 2 +-
src/common/unicode_case.c | 12 +-
src/include/common/unicode_case_table.h | 9 +-
src/include/common/unicode_limits.h | 39 ++++
7 files changed, 202 insertions(+), 71 deletions(-)
create mode 100644 src/include/common/unicode_limits.h
diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c
index 5619daf43c3..5c67c6ffc3a 100644
--- a/src/backend/utils/adt/pg_locale_builtin.c
+++ b/src/backend/utils/adt/pg_locale_builtin.c
@@ -15,11 +15,22 @@
#include "catalog/pg_collation.h"
#include "common/unicode_case.h"
#include "common/unicode_category.h"
+#include "common/unicode_limits.h"
#include "miscadmin.h"
#include "utils/builtins.h"
+#include "utils/memutils.h"
#include "utils/pg_locale.h"
#include "utils/syscache.h"
+/*
+ * The largest text value must fit in MaxAllocSize, but then may grow during
+ * case mapping. While the resulting string will not be representable as a new
+ * text value, we must at least be sure not to overflow a size_t while
+ * processing it.
+ */
+StaticAssertDecl(SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION > MaxAllocSize,
+ "case mapping may overflow size_t");
+
extern pg_locale_t create_pg_locale_builtin(Oid collid,
MemoryContext context);
extern char *get_collation_actual_version_builtin(const char *collcollate);
diff --git a/src/common/unicode/Makefile b/src/common/unicode/Makefile
index c150e403cda..714ccbe00bd 100644
--- a/src/common/unicode/Makefile
+++ b/src/common/unicode/Makefile
@@ -21,7 +21,7 @@ CPPFLAGS += $(ICU_CFLAGS)
# By default, do nothing.
all:
-update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h
+update-unicode: unicode_case_table.h unicode_category_table.h unicode_east_asian_fw_table.h unicode_nonspacing_table.h unicode_norm_hashfunc.h unicode_limits.h unicode_norm_table.h unicode_normprops_table.h unicode_version.h
mv $^ $(top_srcdir)/src/include/common/
$(MAKE) case-check
$(MAKE) category-check
@@ -36,6 +36,8 @@ CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNorma
unicode_version.h: generate-unicode_version.pl
$(PERL) $< --version $(UNICODE_VERSION)
+unicode_limits.h: unicode_case_table.h
+
unicode_case_table.h: generate-unicode_case_table.pl CaseFolding.txt SpecialCasing.txt UnicodeData.txt
$(PERL) $<
@@ -91,4 +93,4 @@ clean:
rm -f $(OBJS) case_test case_test.o category_test category_test.o norm_test norm_test.o
distclean: clean
- rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_norm_table.h
+ rm -f CompositionExclusions.txt CaseFolding.txt DerivedCoreProperties.txt DerivedNormalizationProps.txt EastAsianWidth.txt NormalizationTest.txt PropList.txt SpecialCasing.txt UnicodeData.txt norm_test_table.h unicode_case_table.h unicode_category_table.h unicode_limits.h unicode_norm_table.h
diff --git a/src/common/unicode/generate-unicode_case_table.pl b/src/common/unicode/generate-unicode_case_table.pl
index c1b94f47f56..d1dd3a5e8f2 100644
--- a/src/common/unicode/generate-unicode_case_table.pl
+++ b/src/common/unicode/generate-unicode_case_table.pl
@@ -4,13 +4,14 @@
# or locale-specific mappings.
#
# Input: CaseFolding.txt SpecialCasing.txt UnicodeData.txt
-# Output: unicode_case_table.h
+# Output: unicode_case_table.h unicode_limits.h
#
# Copyright (c) 2000-2026, PostgreSQL Global Development Group
use strict;
use warnings FATAL => 'all';
use Getopt::Long;
+use Encode qw(encode_utf8);
use FindBin;
use lib "$FindBin::RealBin/../../tools/";
@@ -19,11 +20,22 @@ my $output_path = '.';
GetOptions('outdir:s' => \$output_path);
+my $output_limits_file = "$output_path/unicode_limits.h";
my $output_table_file = "$output_path/unicode_case_table.h";
+# The following are computed during table generation:
+
# The maximum number of codepoints that can result from case mapping
-# of a single character. See Unicode section 5.18 "Case Mappings".
-my $MAX_CASE_EXPANSION = 3;
+# of a single character.
+my $UNICODE_MAX_CASEMAP_CODEPOINTS = 0;
+
+# The maximum expansion factor (in bytes) when case mapping a UTF8
+# string.
+my $UTF8_MAX_CASEMAP_EXPANSION = 0;
+
+# The maximum number of UTF8 bytes that can result from case mapping
+# a single code point.
+my $UTF8_CASEMAP_BUFSZ = 0;
my $FH;
@@ -44,15 +56,22 @@ while (my $line = <$FH>)
die "Simple_Titlecase $code out of range" if $simple_titlecase > 0x10FFFF;
die "Simple_Uppercase $code out of range" if $simple_uppercase > 0x10FFFF;
- if ($simple_lowercase || $simple_titlecase || $simple_uppercase)
- {
- $simple{$code} = {
- Simple_Lowercase => ($simple_lowercase || $code),
- Simple_Titlecase => ($simple_titlecase || $code),
- Simple_Uppercase => ($simple_uppercase || $code),
- Simple_Foldcase => $code,
- };
- }
+ next if !$simple_lowercase && !$simple_titlecase && !$simple_uppercase;
+
+ $simple_lowercase ||= $code;
+ $simple_titlecase ||= $code;
+ $simple_uppercase ||= $code;
+
+ update_limits('uppercase', $code, ($simple_uppercase));
+ update_limits('lowercase', $code, ($simple_lowercase));
+ update_limits('titlecase', $code, ($simple_titlecase));
+
+ $simple{$code} = {
+ Simple_Lowercase => $simple_lowercase,
+ Simple_Titlecase => $simple_titlecase,
+ Simple_Uppercase => $simple_uppercase,
+ Simple_Foldcase => $code,
+ };
}
close $FH;
@@ -105,19 +124,9 @@ while (my $line = <$FH>)
push @upper, $code if (scalar @upper == 0);
push @fold, $code;
- # none should map to more than 3 codepoints
- die "lowercase expansion for 0x$elts[0] exceeds maximum: '$elts[1]'"
- if (scalar @lower) > $MAX_CASE_EXPANSION;
- die "titlecase expansion for 0x$elts[0] exceeds maximum: '$elts[2]'"
- if (scalar @title) > $MAX_CASE_EXPANSION;
- die "uppercase expansion for 0x$elts[0] exceeds maximum: '$elts[3]'"
- if (scalar @upper) > $MAX_CASE_EXPANSION;
-
- # pad arrays to a fixed length of 3
- while (scalar @upper < $MAX_CASE_EXPANSION) { push @upper, 0x000000 }
- while (scalar @lower < $MAX_CASE_EXPANSION) { push @lower, 0x000000 }
- while (scalar @title < $MAX_CASE_EXPANSION) { push @title, 0x000000 }
- while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 }
+ update_limits('lowercase', $code, @lower);
+ update_limits('titlecase', $code, @title);
+ update_limits('uppercase', $code, @upper);
# Characters with special mappings may not have simple mappings;
# ensure that an entry exists.
@@ -174,6 +183,8 @@ while (my $line = <$FH>)
die "unsupported status type '$status'"
if $status ne 'S' && $status ne 'C' && $status ne 'F';
+ update_limits('casefold', $code, @fold);
+
# initialize simple case mappings if they don't exist
$simple{$code} ||= {
Simple_Lowercase => $code,
@@ -197,26 +208,12 @@ while (my $line = <$FH>)
if ($status eq 'F' || ($status eq 'C' && defined $special{$code}))
{
- while (scalar @fold < $MAX_CASE_EXPANSION) { push @fold, 0x000000 }
-
#initialize special case mappings if they don't exist
if (!defined $special{$code})
{
my @lower = ($simple{$code}{Simple_Lowercase});
my @title = ($simple{$code}{Simple_Titlecase});
my @upper = ($simple{$code}{Simple_Uppercase});
- while (scalar @lower < $MAX_CASE_EXPANSION)
- {
- push @lower, 0x000000;
- }
- while (scalar @title < $MAX_CASE_EXPANSION)
- {
- push @title, 0x000000;
- }
- while (scalar @upper < $MAX_CASE_EXPANSION)
- {
- push @upper, 0x000000;
- }
$special{$code} = {
Lowercase => \@lower,
Titlecase => \@title,
@@ -245,6 +242,53 @@ die
"special case map contains $num_special entries which cannot be represented in uint8"
if ($num_special > 256);
+open my $LIMITS, '>', $output_limits_file
+ or die "Could not open output file $output_limits_file: $!\n";
+
+print $LIMITS <<"EOS";
+/*-------------------------------------------------------------------------
+ *
+ * unicode_limits.h
+ * Useful limits calculated from Unicode data.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/unicode_limits.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef UNICODE_LIMITS_H
+#define UNICODE_LIMITS_H
+
+/*
+ * File auto-generated by src/common/unicode/generate-unicode_case_table.pl,
+ * do not edit.
+ */
+
+/*
+ * The maximum number of codepoints that can result from case mapping
+ * of a single code point. See Unicode section 5.18 "Case Mappings".
+ */
+#define UNICODE_MAX_CASEMAP_CODEPOINTS $UNICODE_MAX_CASEMAP_CODEPOINTS
+
+/*
+ * The maximum expansion factor of a UTF8 string due to case mapping.
+ */
+#define UTF8_MAX_CASEMAP_EXPANSION $UTF8_MAX_CASEMAP_EXPANSION
+
+/*
+ * The maximum number of UTF8 bytes needed to store the result of case
+ * mapping a single code point.
+ */
+#define UTF8_CASEMAP_BUFSZ $UTF8_CASEMAP_BUFSZ
+
+#endif
+EOS
+
+close $LIMITS;
+
# Start writing out the output files
open my $OT, '>', $output_table_file
or die "Could not open output file $output_table_file: $!\n";
@@ -270,12 +314,7 @@ print $OT <<"EOS";
*/
#include "common/unicode_case.h"
-
-/*
- * The maximum number of codepoints that can result from case mapping
- * of a single character. See Unicode section 5.18 "Case Mappings".
- */
-#define MAX_CASE_EXPANSION 3
+#include "common/unicode_limits.h"
/*
* Case mapping condition flags. For now, only Final_Sigma is supported.
@@ -296,7 +335,7 @@ typedef enum
typedef struct
{
int16 conditions;
- char32_t map[NCaseKind][MAX_CASE_EXPANSION];
+ char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS];
} pg_special_case;
/*
@@ -310,18 +349,26 @@ EOS
foreach my $code (sort { $a <=> $b } (keys %special))
{
- die if scalar @{ $special{$code}{Lowercase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Titlecase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Uppercase} } != $MAX_CASE_EXPANSION;
- die if scalar @{ $special{$code}{Foldcase} } != $MAX_CASE_EXPANSION;
- my $lower = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Lowercase} });
- my $title = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Titlecase} });
- my $upper = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Uppercase} });
- my $fold = join ", ",
- (map { sprintf "0x%06x", $_ } @{ $special{$code}{Foldcase} });
+ my @lower = @{ $special{$code}{Lowercase} };
+ my @title = @{ $special{$code}{Titlecase} };
+ my @upper = @{ $special{$code}{Uppercase} };
+ my @fold = @{ $special{$code}{Foldcase} };
+
+ # pad to fixed length
+ push @lower, 0x000000
+ while (scalar @lower < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @title, 0x000000
+ while (scalar @title < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @upper, 0x000000
+ while (scalar @upper < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+ push @fold, 0x000000
+ while (scalar @fold < $UNICODE_MAX_CASEMAP_CODEPOINTS);
+
+ my $lower = join ", ", (map { sprintf "0x%06x", $_ } @lower);
+ my $title = join ", ", (map { sprintf "0x%06x", $_ } @title);
+ my $upper = join ", ", (map { sprintf "0x%06x", $_ } @upper);
+ my $fold = join ", ", (map { sprintf "0x%06x", $_ } @fold);
+
printf $OT "\t{%s, ", $special{$code}{Conditions};
printf $OT
"{[CaseLower] = {%s},[CaseTitle] = {%s},[CaseUpper] = {%s},[CaseFold] = {%s}}},\n",
@@ -522,6 +569,37 @@ EOS
close $OT;
+# calculate, check, and update limits
+sub update_limits
+{
+ my ($casekind, $codepoint, @mapping) = @_;
+ my $n = scalar @mapping;
+
+ # See Unicode section 5.18 "Case Mappings"
+ my $mapping_str = join ' ', map { sprintf "0x%06x", $_ } @mapping;
+ die sprintf("%s expansion for 0x%06x exceeds maximum: %s",
+ $casekind, $codepoint, $mapping_str)
+ if $n > 3;
+
+ $UNICODE_MAX_CASEMAP_CODEPOINTS = $n
+ if $n > $UNICODE_MAX_CASEMAP_CODEPOINTS;
+
+ my $utf8len1 = length(encode_utf8(chr $codepoint));
+ my $utf8len2 = length(encode_utf8(join '', map { chr } @mapping));
+ my $factor = int(($utf8len1 + $utf8len2 - 1) / $utf8len1);
+ $UTF8_MAX_CASEMAP_EXPANSION = $factor
+ if $factor > $UTF8_MAX_CASEMAP_EXPANSION;
+
+ # Not guaranteed by Unicode standard. If larger than expected,
+ # then evaluate C type overflow risks before using the generated
+ # table.
+ die "UTF8 casemap expansion is greater than 3"
+ if $UTF8_MAX_CASEMAP_EXPANSION > 3;
+
+ $UTF8_CASEMAP_BUFSZ = $utf8len2
+ if $utf8len2 > $UTF8_CASEMAP_BUFSZ;
+}
+
# The function generates C code with a series of nested if-else conditions
# to search for the matching interval.
sub branch
diff --git a/src/common/unicode/meson.build b/src/common/unicode/meson.build
index 8a31fe40f6c..a32d44d1816 100644
--- a/src/common/unicode/meson.build
+++ b/src/common/unicode/meson.build
@@ -28,7 +28,7 @@ update_unicode_common_targets = []
update_unicode_common_targets += \
custom_target('unicode_case_table.h',
input: [unicode_data['CaseFolding.txt'], unicode_data['SpecialCasing.txt'], unicode_data['UnicodeData.txt']],
- output: ['unicode_case_table.h'],
+ output: ['unicode_case_table.h', 'unicode_limits.h'],
command: [
perl, files('generate-unicode_case_table.pl'),
'--outdir', '@OUTDIR@', '@INPUT@'],
diff --git a/src/common/unicode_case.c b/src/common/unicode_case.c
index 399f08235a9..00668a7d4f3 100644
--- a/src/common/unicode_case.c
+++ b/src/common/unicode_case.c
@@ -237,6 +237,12 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
size_t result_len = 0;
size_t boundary = 0;
+ /*
+ * Must be guaranteed by caller to avoid overflow (text values limited to
+ * MaxAllocSize anyway).
+ */
+ Assert(srclen < SIZE_MAX / UTF8_MAX_CASEMAP_EXPANSION);
+
Assert((str_casekind == CaseTitle && wbnext && wbstate) ||
(str_casekind != CaseTitle && !wbnext && !wbstate));
@@ -299,9 +305,9 @@ convert_case(char *dst, size_t dstsize, const char *src, size_t srclen,
}
break;
case CASEMAP_SPECIAL:
- /* replace with up to MAX_CASE_EXPANSION characters */
+ /* replace with up to UNICODE_MAX_CASEMAP_CODEPOINTS */
Assert(simple == 0);
- for (int i = 0; i < MAX_CASE_EXPANSION && special[i]; i++)
+ for (int i = 0; i < UNICODE_MAX_CASEMAP_CODEPOINTS && special[i]; i++)
{
char32_t u2 = special[i];
size_t u2len = unicode_utf8len(u2);
@@ -415,7 +421,7 @@ check_special_conditions(int conditions, const char *str, size_t len,
*
* If full is true, and a special case mapping is found and the conditions are
* met, 'special' is set to the mapping result (which is an array of up to
- * MAX_CASE_EXPANSION characters) and CASEMAP_SPECIAL is returned.
+ * UNICODE_MAX_CASEMAP_CODEPOINTS) and CASEMAP_SPECIAL is returned.
*
* Otherwise, search for a simple mapping, and if found, set 'simple' to the
* result and return CASEMAP_SIMPLE.
diff --git a/src/include/common/unicode_case_table.h b/src/include/common/unicode_case_table.h
index b2274627c4f..9930943453f 100644
--- a/src/include/common/unicode_case_table.h
+++ b/src/include/common/unicode_case_table.h
@@ -18,12 +18,7 @@
*/
#include "common/unicode_case.h"
-
-/*
- * The maximum number of codepoints that can result from case mapping
- * of a single character. See Unicode section 5.18 "Case Mappings".
- */
-#define MAX_CASE_EXPANSION 3
+#include "common/unicode_limits.h"
/*
* Case mapping condition flags. For now, only Final_Sigma is supported.
@@ -44,7 +39,7 @@ typedef enum
typedef struct
{
int16 conditions;
- char32_t map[NCaseKind][MAX_CASE_EXPANSION];
+ char32_t map[NCaseKind][UNICODE_MAX_CASEMAP_CODEPOINTS];
} pg_special_case;
/*
diff --git a/src/include/common/unicode_limits.h b/src/include/common/unicode_limits.h
new file mode 100644
index 00000000000..59e307857ed
--- /dev/null
+++ b/src/include/common/unicode_limits.h
@@ -0,0 +1,39 @@
+/*-------------------------------------------------------------------------
+ *
+ * unicode_limits.h
+ * Useful limits calculated from Unicode data.
+ *
+ * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/unicode_limits.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef UNICODE_LIMITS_H
+#define UNICODE_LIMITS_H
+
+/*
+ * File auto-generated by src/common/unicode/generate-unicode_case_table.pl,
+ * do not edit.
+ */
+
+/*
+ * The maximum number of codepoints that can result from case mapping
+ * of a single code point. See Unicode section 5.18 "Case Mappings".
+ */
+#define UNICODE_MAX_CASEMAP_CODEPOINTS 3
+
+/*
+ * The maximum expansion factor of a UTF8 string due to case mapping.
+ */
+#define UTF8_MAX_CASEMAP_EXPANSION 3
+
+/*
+ * The maximum number of UTF8 bytes needed to store the result of case
+ * mapping a single code point.
+ */
+#define UTF8_CASEMAP_BUFSZ 6
+
+#endif
--
2.43.0
end of thread, other threads:[~2026-07-16 21:47 UTC | newest]
Thread overview: 27+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2025-12-12 23:24 Small patch to improve safety of utf8_to_unicode(). Jeff Davis <pgsql@j-davis.com>
2025-12-13 23:22 ` Chao Li <li.evan.chao@gmail.com>
2025-12-15 20:23 ` Jeff Davis <pgsql@j-davis.com>
2025-12-15 23:34 ` Chao Li <li.evan.chao@gmail.com>
2025-12-17 19:37 ` Jeff Davis <pgsql@j-davis.com>
2026-06-19 23:22 ` Jeff Davis <pgsql@j-davis.com>
2026-06-23 02:02 ` Jeff Davis <pgsql@j-davis.com>
2026-06-24 05:45 ` Jeff Davis <pgsql@j-davis.com>
2026-06-24 08:44 ` Chao Li <li.evan.chao@gmail.com>
2026-06-24 21:57 ` Jeff Davis <pgsql@j-davis.com>
2026-06-25 04:10 ` Chao Li <li.evan.chao@gmail.com>
2026-06-25 17:38 ` Jeff Davis <pgsql@j-davis.com>
2026-06-26 04:38 ` Chao Li <li.evan.chao@gmail.com>
2026-06-27 14:02 ` Jeff Davis <pgsql@j-davis.com>
2026-06-28 22:29 ` Chao Li <li.evan.chao@gmail.com>
2026-07-08 00:56 ` Jeff Davis <pgsql@j-davis.com>
2026-07-08 16:48 ` Chao Li <li.evan.chao@gmail.com>
2026-07-13 13:42 ` Tom Lane <tgl@sss.pgh.pa.us>
2026-07-13 17:09 ` Jeff Davis <pgsql@j-davis.com>
2026-07-13 17:19 ` Tom Lane <tgl@sss.pgh.pa.us>
2026-07-13 17:20 ` Tom Lane <tgl@sss.pgh.pa.us>
2026-07-14 03:26 ` Jeff Davis <pgsql@j-davis.com>
2026-07-15 21:39 ` Jeff Davis <pgsql@j-davis.com>
2026-07-15 22:25 ` Tom Lane <tgl@sss.pgh.pa.us>
2026-07-16 21:47 ` Jeff Davis <pgsql@j-davis.com>
2026-06-24 09:29 ` Ayush Tiwari <ayushtiwari.slg01@gmail.com>
2026-06-24 23:43 ` Jeff Davis <pgsql@j-davis.com>
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox