agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
32+ messages / 5 participants
[nested] [flat]

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Something is wrong with wal_compression
@ 2023-01-26 19:43  Tom Lane <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Tom Lane @ 2023-01-26 19:43 UTC (permalink / raw)
  To: [email protected]; +Cc: Michael Paquier <[email protected]>

The symptom being exhibited by Michael's new BF animal tanager
is perfectly reproducible elsewhere.

$ cat /home/postgres/tmp/temp_config
#default_toast_compression = lz4
wal_compression = lz4
$ export TEMP_CONFIG=/home/postgres/tmp/temp_config
$ cd ~/pgsql/src/test/recovery
$ make check PROVE_TESTS=t/011_crash_recovery.pl
...
+++ tap check in src/test/recovery +++
t/011_crash_recovery.pl .. 1/? 
#   Failed test 'new xid after restart is greater'
#   at t/011_crash_recovery.pl line 53.
#     '729'
#         >
#     '729'

#   Failed test 'xid is aborted after crash'
#   at t/011_crash_recovery.pl line 57.
#          got: 'committed'
#     expected: 'aborted'
# Looks like you failed 2 tests of 3.

Maybe this is somehow the test script's fault, but I don't see how.

It fails the same way with 'wal_compression = pglz', so I think it's
generic to that whole feature rather than specific to LZ4.

			regards, tom lane






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 20:08  Justin Pryzby <[email protected]>
  parent: Tom Lane <[email protected]>
  0 siblings, 2 replies; 32+ messages in thread

From: Justin Pryzby @ 2023-01-26 20:08 UTC (permalink / raw)
  To: Tom Lane <[email protected]>; +Cc: [email protected]; Michael Paquier <[email protected]>

On Thu, Jan 26, 2023 at 02:43:29PM -0500, Tom Lane wrote:
> The symptom being exhibited by Michael's new BF animal tanager
> is perfectly reproducible elsewhere.

I think these tests have always failed with wal_compression ?

https://www.postgresql.org/message-id/20210308.173242.463790587797836129.horikyota.ntt%40gmail.com
https://www.postgresql.org/message-id/[email protected]
https://www.postgresql.org/message-id/[email protected]

https://www.postgresql.org/message-id/[email protected]
|My buildfarm machine has been changed to use wal_compression = lz4,
|while on it for HEAD runs.







^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 20:12  Tom Lane <[email protected]>
  parent: Justin Pryzby <[email protected]>
  1 sibling, 1 reply; 32+ messages in thread

From: Tom Lane @ 2023-01-26 20:12 UTC (permalink / raw)
  To: Justin Pryzby <[email protected]>; +Cc: [email protected]; Michael Paquier <[email protected]>

Justin Pryzby <[email protected]> writes:
> On Thu, Jan 26, 2023 at 02:43:29PM -0500, Tom Lane wrote:
>> The symptom being exhibited by Michael's new BF animal tanager
>> is perfectly reproducible elsewhere.

> I think these tests have always failed with wal_compression ?

If that's a known problem, and we've done nothing about it,
that is pretty horrid.  That test case is demonstrating fundamental
database corruption after a crash.

			regards, tom lane






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 20:15  Justin Pryzby <[email protected]>
  parent: Justin Pryzby <[email protected]>
  1 sibling, 0 replies; 32+ messages in thread

From: Justin Pryzby @ 2023-01-26 20:15 UTC (permalink / raw)
  To: Tom Lane <[email protected]>; +Cc: [email protected]; Michael Paquier <[email protected]>

On Thu, Jan 26, 2023 at 02:08:27PM -0600, Justin Pryzby wrote:
> On Thu, Jan 26, 2023 at 02:43:29PM -0500, Tom Lane wrote:
> > The symptom being exhibited by Michael's new BF animal tanager
> > is perfectly reproducible elsewhere.
> 
> I think these tests have always failed with wal_compression ?
> 
> https://www.postgresql.org/message-id/20210308.173242.463790587797836129.horikyota.ntt%40gmail.com
> https://www.postgresql.org/message-id/[email protected]
> https://www.postgresql.org/message-id/[email protected]

+ https://www.postgresql.org/message-id/c86ce84f-dd38-9951-102f-13a931210f52%40dunslane.net






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 21:28  Andrey Borodin <[email protected]>
  parent: Tom Lane <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Andrey Borodin @ 2023-01-26 21:28 UTC (permalink / raw)
  To: Tom Lane <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

On Thu, Jan 26, 2023 at 12:12 PM Tom Lane <[email protected]> wrote:
>
> That test case is demonstrating fundamental
> database corruption after a crash.
>

Not exactly corruption. XID was not persisted and buffer data did not
hit a disk. Database is in the correct state.

It was discussed long before WAL compression here [0]. The thing is it
is easier to reproduce with compression, but compression has nothing
to do with it, as far as I understand.

Proposed fix is here[1], but I think it's better to fix the test. It
should not veryfi Xid, but rather side effects of "CREATE TABLE mine(x
integer);".


Best regards, Andrey Borodin.

[0] https://www.postgresql.org/message-id/flat/565FB155-C6B0-41E2-8C44-7B514DC25132%2540yandex-team.ru
[1] https://www.postgresql.org/message-id/flat/20210313012820.GJ29463%40telsasoft.com#0f18d3a4d593ea656f...






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 22:14  Tom Lane <[email protected]>
  parent: Andrey Borodin <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Tom Lane @ 2023-01-26 22:14 UTC (permalink / raw)
  To: Andrey Borodin <[email protected]>; +Cc: Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

Andrey Borodin <[email protected]> writes:
> On Thu, Jan 26, 2023 at 12:12 PM Tom Lane <[email protected]> wrote:
>> That test case is demonstrating fundamental
>> database corruption after a crash.

> Not exactly corruption. XID was not persisted and buffer data did not
> hit a disk. Database is in the correct state.

Really?  I don't see how this part is even a little bit okay:

[00:40:50.744](0.046s) not ok 3 - xid is aborted after crash
[00:40:50.745](0.001s) 
[00:40:50.745](0.000s) #   Failed test 'xid is aborted after crash'
#   at t/011_crash_recovery.pl line 57.
[00:40:50.746](0.001s) #          got: 'committed'
#     expected: 'aborted'

If any tuples made by that transaction had reached disk,
we'd have a problem.

			regards, tom lane






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 22:50  Thomas Munro <[email protected]>
  parent: Tom Lane <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Thomas Munro @ 2023-01-26 22:50 UTC (permalink / raw)
  To: Tom Lane <[email protected]>; +Cc: Andrey Borodin <[email protected]>; Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

On Fri, Jan 27, 2023 at 11:14 AM Tom Lane <[email protected]> wrote:
> Andrey Borodin <[email protected]> writes:
> > On Thu, Jan 26, 2023 at 12:12 PM Tom Lane <[email protected]> wrote:
> >> That test case is demonstrating fundamental
> >> database corruption after a crash.
>
> > Not exactly corruption. XID was not persisted and buffer data did not
> > hit a disk. Database is in the correct state.
>
> Really?  I don't see how this part is even a little bit okay:
>
> [00:40:50.744](0.046s) not ok 3 - xid is aborted after crash
> [00:40:50.745](0.001s)
> [00:40:50.745](0.000s) #   Failed test 'xid is aborted after crash'
> #   at t/011_crash_recovery.pl line 57.
> [00:40:50.746](0.001s) #          got: 'committed'
> #     expected: 'aborted'
>
> If any tuples made by that transaction had reached disk,
> we'd have a problem.

The problem is that the WAL wasn't flushed, allowing the same xid to
be allocated again after crash recovery.  But for any data pages to
hit the disk, we'd have to flush WAL first, so then it couldn't
happen, no?  FWIW I also re-complained about the dangers of anyone
relying on pg_xact_status() for its stated purpose after seeing
tanager's failure[1].

[1] https://www.postgresql.org/message-id/CA%2BhUKGJ9p2JPPMA4eYAKq%3Dr9d_4_8vziet_tS1LEBbiny5-ypA%40mail...






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-26 23:04  Tom Lane <[email protected]>
  parent: Thomas Munro <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Tom Lane @ 2023-01-26 23:04 UTC (permalink / raw)
  To: Thomas Munro <[email protected]>; +Cc: Andrey Borodin <[email protected]>; Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

Thomas Munro <[email protected]> writes:
> On Fri, Jan 27, 2023 at 11:14 AM Tom Lane <[email protected]> wrote:
>> If any tuples made by that transaction had reached disk,
>> we'd have a problem.

> The problem is that the WAL wasn't flushed, allowing the same xid to
> be allocated again after crash recovery.  But for any data pages to
> hit the disk, we'd have to flush WAL first, so then it couldn't
> happen, no?

Ah, now I get the point: the "committed xact" seen after restart
isn't the same one as we saw before the crash, but a new one that
was given the same XID because nothing about the old one had made
it to disk yet.

> FWIW I also re-complained about the dangers of anyone
> relying on pg_xact_status() for its stated purpose after seeing
> tanager's failure[1].

Indeed, it seems like this behavior makes pg_xact_status() basically
useless as things stand.

			regards, tom lane






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-27 00:14  Andrey Borodin <[email protected]>
  parent: Tom Lane <[email protected]>
  0 siblings, 1 reply; 32+ messages in thread

From: Andrey Borodin @ 2023-01-27 00:14 UTC (permalink / raw)
  To: Tom Lane <[email protected]>; +Cc: Thomas Munro <[email protected]>; Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

On Thu, Jan 26, 2023 at 3:04 PM Tom Lane <[email protected]> wrote:
>
> Indeed, it seems like this behavior makes pg_xact_status() basically
> useless as things stand.
>

If we agree that xid allocation is not something persistent, let's fix
the test? We can replace a check with select * from pg_class or,
maybe, add an amcheck run.
As far as I recollect, this test was introduced to test this new
function in 857ee8e391f.


Best regards, Andrey Borodin.






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

* Re: Something is wrong with wal_compression
@ 2023-01-27 00:23  Tom Lane <[email protected]>
  parent: Andrey Borodin <[email protected]>
  0 siblings, 0 replies; 32+ messages in thread

From: Tom Lane @ 2023-01-27 00:23 UTC (permalink / raw)
  To: Andrey Borodin <[email protected]>; +Cc: Thomas Munro <[email protected]>; Justin Pryzby <[email protected]>; [email protected]; Michael Paquier <[email protected]>

Andrey Borodin <[email protected]> writes:
> On Thu, Jan 26, 2023 at 3:04 PM Tom Lane <[email protected]> wrote:
>> Indeed, it seems like this behavior makes pg_xact_status() basically
>> useless as things stand.

> If we agree that xid allocation is not something persistent, let's fix
> the test?

If we're not going to fix this behavior, we need to fix the docs
to disclaim that pg_xact_status() is of use for what it's said
to be good for.

			regards, tom lane






^ permalink  raw  reply  [nested|flat] 32+ messages in thread

end of thread, other threads:[~2023-01-27 00:23 UTC | newest]

Thread overview: 32+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2023-01-26 19:43 Something is wrong with wal_compression Tom Lane <[email protected]>
2023-01-26 20:08 ` Re: Something is wrong with wal_compression Justin Pryzby <[email protected]>
2023-01-26 20:12   ` Re: Something is wrong with wal_compression Tom Lane <[email protected]>
2023-01-26 21:28     ` Re: Something is wrong with wal_compression Andrey Borodin <[email protected]>
2023-01-26 22:14       ` Re: Something is wrong with wal_compression Tom Lane <[email protected]>
2023-01-26 22:50         ` Re: Something is wrong with wal_compression Thomas Munro <[email protected]>
2023-01-26 23:04           ` Re: Something is wrong with wal_compression Tom Lane <[email protected]>
2023-01-27 00:14             ` Re: Something is wrong with wal_compression Andrey Borodin <[email protected]>
2023-01-27 00:23               ` Re: Something is wrong with wal_compression Tom Lane <[email protected]>
2023-01-26 20:15   ` Re: Something is wrong with wal_compression Justin Pryzby <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox