agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
23+ messages / 2 participants
[nested] [flat]

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature.
@ 2021-02-01 15:30  Heikki Linnakangas <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Heikki Linnakangas @ 2021-02-01 15:30 UTC (permalink / raw)

Per John Naylor's report, and some bugs I found while testing.

TODO: to be squashed.
---
 src/backend/utils/mb/conv.c                      | 16 ++++++----------
 .../utf8_and_iso8859_1/utf8_and_iso8859_1.c      |  8 ++++++++
 src/backend/utils/mb/mbutils.c                   |  5 +++--
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/backend/utils/mb/conv.c b/src/backend/utils/mb/conv.c
index b83358bc7a5..f3bda5753d0 100644
--- a/src/backend/utils/mb/conv.c
+++ b/src/backend/utils/mb/conv.c
@@ -497,7 +497,6 @@ UtfToLocal(const unsigned char *utf, int len,
 	int			l;
 	const pg_utf_to_local_combined *cp;
 	const unsigned char *start = utf;
-	const unsigned char *cur = utf;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -511,8 +510,6 @@ UtfToLocal(const unsigned char *utf, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*utf == '\0')
 			break;
@@ -648,10 +645,11 @@ UtfToLocal(const unsigned char *utf, int len,
 		}
 
 		/* failed to translate this character */
+		utf -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(PG_UTF8, encoding,
-								   (const char *) (utf - l), len);
+								   (const char *) utf, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -660,7 +658,7 @@ UtfToLocal(const unsigned char *utf, int len,
 
 	*iso = '\0';
 
-	return cur - start;
+	return utf - start;
 }
 
 /*
@@ -701,7 +699,6 @@ LocalToUtf(const unsigned char *iso, int len,
 	int			l;
 	const pg_local_to_utf_combined *cp;
 	const unsigned char *start = iso;
-	const unsigned char *cur = iso;
 
 	if (!PG_VALID_ENCODING(encoding))
 		ereport(ERROR,
@@ -715,8 +712,6 @@ LocalToUtf(const unsigned char *iso, int len,
 		unsigned char b3 = 0;
 		unsigned char b4 = 0;
 
-		cur = iso;
-
 		/* "break" cases all represent errors */
 		if (*iso == '\0')
 			break;
@@ -799,10 +794,11 @@ LocalToUtf(const unsigned char *iso, int len,
 		}
 
 		/* failed to translate this character */
+		iso -= l;
 		if (noError)
 			break;
 		report_untranslatable_char(encoding, PG_UTF8,
-								   (const char *) (iso - l), len);
+								   (const char *) iso, len);
 	}
 
 	/* if we broke out of loop early, must be invalid input */
@@ -811,5 +807,5 @@ LocalToUtf(const unsigned char *iso, int len,
 
 	*utf = '\0';
 
-	return cur - start;
+	return iso - start;
 }
diff --git a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
index 8ac93604a1b..d0dc4cca378 100644
--- a/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
+++ b/src/backend/utils/mb/conversion_procs/utf8_and_iso8859_1/utf8_and_iso8859_1.c
@@ -104,10 +104,18 @@ utf8_to_iso8859_1(PG_FUNCTION_ARGS)
 			int			l = pg_utf_mblen(src);
 
 			if (l > len || !pg_utf8_islegal(src, l))
+			{
+				if (noError)
+					break;
 				report_invalid_encoding(PG_UTF8, (const char *) src, len);
+			}
 			if (l != 2)
+			{
+				if (noError)
+					break;
 				report_untranslatable_char(PG_UTF8, PG_LATIN1,
 										   (const char *) src, len);
+			}
 			c1 = src[1] & 0x3f;
 			c = ((c & 0x1f) << 6) | c1;
 			if (c >= 0x80 && c <= 0xff)
diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c
index 3e106027d75..af1cafe523a 100644
--- a/src/backend/utils/mb/mbutils.c
+++ b/src/backend/utils/mb/mbutils.c
@@ -818,12 +818,13 @@ perform_default_encoding_conversion(const char *src, int len,
 		MemoryContextAllocHuge(CurrentMemoryContext,
 							   (Size) len * MAX_CONVERSION_GROWTH + 1);
 
-	FunctionCall5(flinfo,
+	FunctionCall6(flinfo,
 				  Int32GetDatum(src_encoding),
 				  Int32GetDatum(dest_encoding),
 				  CStringGetDatum(src),
 				  CStringGetDatum(result),
-				  Int32GetDatum(len));
+				  Int32GetDatum(len),
+				  BoolGetDatum(false));
 
 	/*
 	 * Release extra space if there might be a lot --- see comments in
-- 
2.29.2


--------------39E46EF15EBFAA37D53332E5
Content-Type: text/x-patch; charset=UTF-8;
 name="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
 filename*0="v3-0005-Do-COPY-FROM-encoding-conversion-verification-in-.pa";
 filename*1="tch"



^ permalink  raw  reply  [nested|flat] 23+ messages in thread

* [PATCH 2/2] Remove extra code block.
@ 2022-07-01 02:40  Kyotaro Horiguchi <[email protected]>
  0 siblings, 0 replies; 23+ messages in thread

From: Kyotaro Horiguchi @ 2022-07-01 02:40 UTC (permalink / raw)

---
 src/backend/replication/basebackup.c | 185 +++++++++++++--------------
 1 file changed, 91 insertions(+), 94 deletions(-)

diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index e4345dbff2..ec9bc6f52b 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -233,6 +233,8 @@ perform_base_backup(basebackup_options *opt, bbsink *sink)
 	StringInfo	labelfile;
 	StringInfo	tblspc_map_file;
 	backup_manifest_info manifest;
+	ListCell   *lc;
+	tablespaceinfo *ti;
 
 	/* Initial backup state, insofar as we know it now. */
 	state.tablespaces = NIL;
@@ -263,114 +265,109 @@ perform_base_backup(basebackup_options *opt, bbsink *sink)
 										labelfile, &state.tablespaces,
 										tblspc_map_file);
 
+	/* Add a node for the base directory at the end */
+	ti = palloc0(sizeof(tablespaceinfo));
+	ti->size = -1;
+	state.tablespaces = lappend(state.tablespaces, ti);
+
+	/*
+	 * Calculate the total backup size by summing up the size of each
+	 * tablespace
+	 */
+	if (opt->progress)
 	{
-		ListCell   *lc;
-		tablespaceinfo *ti;
-
-		/* Add a node for the base directory at the end */
-		ti = palloc0(sizeof(tablespaceinfo));
-		ti->size = -1;
-		state.tablespaces = lappend(state.tablespaces, ti);
-
-		/*
-		 * Calculate the total backup size by summing up the size of each
-		 * tablespace
-		 */
-		if (opt->progress)
-		{
-			basebackup_progress_estimate_backup_size();
-
-			foreach(lc, state.tablespaces)
-			{
-				tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
-
-				if (tmp->path == NULL)
-					tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
-										true, NULL, NULL);
-				else
-					tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
-											   NULL);
-				state.bytes_total += tmp->size;
-			}
-			state.bytes_total_is_valid = true;
-		}
-
-		/* notify basebackup sink about start of backup */
-		bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
-
-		/* Send off our tablespaces one by one */
+		basebackup_progress_estimate_backup_size();
+		
 		foreach(lc, state.tablespaces)
 		{
-			tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
+			tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
+			
+			if (tmp->path == NULL)
+				tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
+									true, NULL, NULL);
+			else
+				tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
+										   NULL);
+			state.bytes_total += tmp->size;
+		}
+		state.bytes_total_is_valid = true;
+	}
 
-			if (ti->path == NULL)
+	/* notify basebackup sink about start of backup */
+	bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
+
+	/* Send off our tablespaces one by one */
+	foreach(lc, state.tablespaces)
+	{
+		tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
+
+		if (ti->path == NULL)
+		{
+			struct stat statbuf;
+			bool		sendtblspclinks = true;
+
+			bbsink_begin_archive(sink, "base.tar");
+
+			/* In the main tar, include the backup_label first... */
+			sendFileWithContent(sink, BACKUP_LABEL_FILE, labelfile->data,
+								&manifest);
+
+			/* Then the tablespace_map file, if required... */
+			if (opt->sendtblspcmapfile)
 			{
-				struct stat statbuf;
-				bool		sendtblspclinks = true;
-
-				bbsink_begin_archive(sink, "base.tar");
-
-				/* In the main tar, include the backup_label first... */
-				sendFileWithContent(sink, BACKUP_LABEL_FILE, labelfile->data,
+				sendFileWithContent(sink, TABLESPACE_MAP, tblspc_map_file->data,
 									&manifest);
-
-				/* Then the tablespace_map file, if required... */
-				if (opt->sendtblspcmapfile)
-				{
-					sendFileWithContent(sink, TABLESPACE_MAP, tblspc_map_file->data,
-										&manifest);
-					sendtblspclinks = false;
-				}
-
-				/* Then the bulk of the files... */
-				sendDir(sink, ".", 1, false, state.tablespaces,
-						sendtblspclinks, &manifest, NULL);
-
-				/* ... and pg_control after everything else. */
-				if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
-					ereport(ERROR,
-							(errcode_for_file_access(),
-							 errmsg("could not stat file \"%s\": %m",
-									XLOG_CONTROL_FILE)));
-				sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
-						 false, InvalidOid, &manifest, NULL);
+				sendtblspclinks = false;
 			}
-			else
-			{
-				char	   *archive_name = psprintf("%s.tar", ti->oid);
 
-				bbsink_begin_archive(sink, archive_name);
-
-				sendTablespace(sink, ti->path, ti->oid, false, &manifest);
-			}
+			/* Then the bulk of the files... */
+			sendDir(sink, ".", 1, false, state.tablespaces,
+					sendtblspclinks, &manifest, NULL);
+
+			/* ... and pg_control after everything else. */
+			if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
+				ereport(ERROR,
+						(errcode_for_file_access(),
+						 errmsg("could not stat file \"%s\": %m",
+								XLOG_CONTROL_FILE)));
+			sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
+					 false, InvalidOid, &manifest, NULL);
+		}
+		else
+		{
+			char	   *archive_name = psprintf("%s.tar", ti->oid);
 
-			/*
-			 * If we're including WAL, and this is the main data directory we
-			 * don't treat this as the end of the tablespace. Instead, we will
-			 * include the xlog files below and stop afterwards. This is safe
-			 * since the main data directory is always sent *last*.
-			 */
-			if (opt->includewal && ti->path == NULL)
-			{
-				Assert(lnext(state.tablespaces, lc) == NULL);
-			}
-			else
-			{
-				/* Properly terminate the tarfile. */
-				StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
-								 "BLCKSZ too small for 2 tar blocks");
-				memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
-				bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
+			bbsink_begin_archive(sink, archive_name);
 
-				/* OK, that's the end of the archive. */
-				bbsink_end_archive(sink);
-			}
+			sendTablespace(sink, ti->path, ti->oid, false, &manifest);
 		}
 
-		basebackup_progress_wait_wal_archive(&state);
-		endptr = do_pg_backup_stop(labelfile->data, !opt->nowait, &endtli);
+		/*
+		 * If we're including WAL, and this is the main data directory we
+		 * don't treat this as the end of the tablespace. Instead, we will
+		 * include the xlog files below and stop afterwards. This is safe
+		 * since the main data directory is always sent *last*.
+		 */
+		if (opt->includewal && ti->path == NULL)
+		{
+			Assert(lnext(state.tablespaces, lc) == NULL);
+		}
+		else
+		{
+			/* Properly terminate the tarfile. */
+			StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
+							 "BLCKSZ too small for 2 tar blocks");
+			memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
+			bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE);
+
+			/* OK, that's the end of the archive. */
+			bbsink_end_archive(sink);
+		}
 	}
 
+	basebackup_progress_wait_wal_archive(&state);
+	endptr = do_pg_backup_stop(labelfile->data, !opt->nowait, &endtli);
+
 	if (opt->includewal)
 	{
 		/*
-- 
2.31.1


----Next_Part(Fri_Jul__1_11_46_53_2022_957)----





^ permalink  raw  reply  [nested|flat] 23+ messages in thread

end of thread, other threads:[~2022-07-01 02:40 UTC | newest]

Thread overview: 23+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2021-02-01 15:30 [PATCH v3 4/5] Fix bugs in the commit to change conversion function signature. Heikki Linnakangas <[email protected]>
2022-07-01 02:40 [PATCH 2/2] Remove extra code block. Kyotaro Horiguchi <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox